From 70b8c37de7d8698325971d4dc1338b71a82b0015 Mon Sep 17 00:00:00 2001
From: boring_nick <boring-nick@users.noreply.github.com>
Date: Sat, 10 Jun 2023 14:39:06 +0300
Subject: [PATCH] add docs

---
 .gitignore          |  1 +
 README.md           | 49 +++++++++++++++++++++++++++++++++++++++++++++
 config.example.json | 11 ++++++++++
 docs/MIGRATION.md   | 25 +++++++++++++++++++++++
 4 files changed, 86 insertions(+)
 create mode 100644 README.md
 create mode 100644 config.example.json
 create mode 100644 docs/MIGRATION.md

diff --git a/.gitignore b/.gitignore
index f46b1b2..1cf90e0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 /target
 /logs*
 config.json
+/ch-data
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..9f3b8af
--- /dev/null
+++ b/README.md
@@ -0,0 +1,49 @@
+# Rustlog
+
+## Description
+Rustlog is a Twitch logging service based on [justlog](https://github.com/gempir/justlog). It provides the same web UI and API, but it utilizes [Clickhouse](https://clickhouse.com) for storage instead of text files.
+
+
+## Installation
+
+Create a `config.json` file (see [config.example.json](./config.example.json))
+
+### Docker
+```yaml
+version: "3.8"
+  
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse
+    volumes:
+      - "./ch-data:/var/lib/clickhouse:rw"
+    environment:
+      CLICKHOUSE_DB: "rustlog"
+    restart: unless-stopped
+        
+  rustlog:
+    image: ghcr.io/boring-nick/rustlog:master
+    container_name: rustlog
+    ports:
+      - 8025:8025 
+    volumes:
+      - "./config.json:/config.json"
+    depends_on: 
+      - clickhouse
+    restart: unless-stopped
+```
+
+### From source
+
+- Set up Clickhouse
+- `cargo install --locked --git https://github.com/boring-nick/rustlog`
+- You can now run the `rustlog` binary
+
+## Advantages over justlog
+
+- Significantly better storage efficiency (2x+ improvement) thanks to not duplicating log files and better compression (using ZSTD in Clickhouse)
+- Blazing fast log queries with response streaming and a [highly performant IRC parser](https://github.com/jprochazk/twitch-rs)
+
+## Migrating from justlog
+See [MIGRATION.md](./docs/MIGRATION.md)
diff --git a/config.example.json b/config.example.json
new file mode 100644
index 0000000..6c96c20
--- /dev/null
+++ b/config.example.json
@@ -0,0 +1,11 @@
+{
+  "clickhouseUrl": "http://clickhouse:8123",
+  "clickhouseDb": "rustlog",
+  "clickhouseUsername": null,
+  "clickhousePassword": null,
+  "listenAddress": "0.0.0.0:8025",
+  "channels": ["12345"],
+  "clientID": "id",
+  "clientSecret": "secret",
+  "admins": []
+}
\ No newline at end of file
diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md
new file mode 100644
index 0000000..af56c70
--- /dev/null
+++ b/docs/MIGRATION.md
@@ -0,0 +1,25 @@
+# Migrating from justlog
+
+Rustlog supports migrating your existing data from justlog. This process will read all log files and write them into the database.
+
+## Config
+Rustlog uses a config format nearly identical to justlog, however you still need to add Clickhouse connection settings to it. See [config.example.json](../config.example.json) for the keys starting with `clickhouse`.
+
+After this, you should have a running rustlog instance logging new messages.
+
+## Data
+First, rustlog needs to have access to the justlog logs directory. If using docker, you need to add it as a volume mount to the container.
+
+After that, you can run the migration command.
+
+Docker:
+```
+docker exec -it rustlog rustlog migrate --source-dir /logs --jobs 1
+```
+Manual installation:
+```
+rustlog migrate --source-dir /path/to/logs --jobs 1
+```
+The `--jobs` parameter defines how many threads rustlog will use for migrating. If your logs are on a HDD, you should keep it at 1, as IO will likely be the bottleneck anyway. If you have an SSD, then setting the value to half of your CPU threads should generally work well.
+
+The migration can take anywhere from a few minutes to a few hours depending on your amount of logs and system resources.