From c3f18f8b34bd8403a040a263bc8354bc7a9b4eae Mon Sep 17 00:00:00 2001 From: journey-ad Date: Mon, 6 Mar 2023 18:57:41 +0800 Subject: [PATCH] perf: Delayed writing to database - Implement delayed writing feature to database for improved performance and reduced write frequency - Update docs --- .gitignore | 1 + Readme.md | 22 +++++++++---------- config.yml | 2 +- db/mongodb.js | 18 +++++++++++++++- db/sqlite.js | 57 ++++++++++++++++++++++++++++--------------------- index.js | 45 +++++++++++++++++++++++++++++++++----- package.json | 6 +++--- views/index.pug | 4 ++-- 8 files changed, 108 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index cb9c106..b329f95 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. +count.db /.history # dependencies diff --git a/Readme.md b/Readme.md index 1164e8f..a521391 100644 --- a/Readme.md +++ b/Readme.md @@ -1,8 +1,8 @@ -# Moe-counter +# Moe-Counter 多种风格可选的萌萌计数器 -![Moe-counter](https://count.getloli.com/get/@Moe-counter.github) +![Moe-Counter](https://count.getloli.com/get/@Moe-counter.github)
More theme @@ -26,17 +26,17 @@ ### Install -#### Run on Repl.it +#### Run on Replit -- Open the url [https://repl.it/@journeyad/Moe-counter](https://repl.it/@journeyad/Moe-counter) +- Open the url [https://replit.com/@journeyad/Moe-Counter](https://replit.com/@journeyad/Moe-Counter) - Just hit the **Fork** button - And hit the **Run** button #### Deploying on your own server ```shell -$ git clone https://github.com/journey-ad/Moe-counter.git -$ cd Moe-counter +$ git clone https://github.com/journey-ad/Moe-Counter.git +$ cd Moe-Counter $ yarn install $ yarn start @@ -51,7 +51,7 @@ app: port: 3000 db: - type: mongodb # sqlite or mongodb + type: sqlite # sqlite or mongodb ``` If you use mongodb, you need to specify the environment variable `DB_URL` @@ -61,7 +61,7 @@ If you use mongodb, you need to specify the environment variable `DB_URL` export DB_URL=mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count ``` -repl.it can use `.env` file, [documentation](https://docs.repl.it/repls/secret-keys) +replit can use Secrets, [documentation](https://docs.replit.com/programming-ide/storing-sensitive-information-environment-variables) ``` DB_URL="mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count" @@ -69,8 +69,8 @@ DB_URL="mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count" ## Credits -* [repl.it](https://repl.it/) -* [A-SOUL](https://www.asoulworld.com/) (非官方导航站) +* [replit](https://replit.com/) +* [A-SOUL_Official](https://space.bilibili.com/703007996) * [moebooru](https://github.com/moebooru/moebooru) * rule34.xxx NSFW * gelbooru.com NSFW @@ -78,4 +78,4 @@ DB_URL="mongodb+srv://account:passwd@***.***.***.mongodb.net/db_count" ## License -[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-counter.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-counter?ref=badge_large) +[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-Counter.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fjourney-ad%2FMoe-Counter?ref=badge_large) diff --git a/config.yml b/config.yml index f63ed72..6e6bc78 100644 --- a/config.yml +++ b/config.yml @@ -2,4 +2,4 @@ app: port: 3000 db: - type: mongodb # sqlite or mongodb \ No newline at end of file + type: sqlite # sqlite or mongodb diff --git a/db/mongodb.js b/db/mongodb.js index 57f1b72..4119c7a 100644 --- a/db/mongodb.js +++ b/db/mongodb.js @@ -32,8 +32,24 @@ function setNum(name, num) { .exec() } +function setNumMulti(counters) { + const bulkOps = counters.map(obj => { + const { name, num } = obj + return { + updateOne: { + filter: { name }, + update: { name, num }, + upsert: true + } + } + }) + + return Count.bulkWrite(bulkOps, { ordered : false }) +} + module.exports = { getNum, getAll, - setNum + setNum, + setNumMulti } diff --git a/db/sqlite.js b/db/sqlite.js index 0912101..4b09dd1 100644 --- a/db/sqlite.js +++ b/db/sqlite.js @@ -1,11 +1,11 @@ 'use strict' const path = require('path') -const sqlite3 = require('sqlite3') +const Database = require('better-sqlite3') -const db = new sqlite3.Database(path.resolve(__dirname, '../count.db')) +const db = new Database(path.resolve(__dirname, '../count.db')) -db.run(`CREATE TABLE IF NOT EXISTS tb_count ( +db.exec(`CREATE TABLE IF NOT EXISTS tb_count ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE, @@ -17,44 +17,53 @@ db.run(`CREATE TABLE IF NOT EXISTS tb_count ( function getNum(name) { return new Promise((resolve, reject) => { - db.get('SELECT `name`, `num` from tb_count WHERE `name` = ?', name, (err, row) => { - if (err) reject(err) - - resolve(row || { name, num: 0 }) - }) + const stmt = db.prepare('SELECT `name`, `num` from tb_count WHERE `name` = ?') + const row = stmt.get(name) + resolve(row || { name, num: 0 }) }) } function getAll(name) { return new Promise((resolve, reject) => { - db.get('SELECT * from tb_count', (err, row) => { - if (err) reject(err) - - resolve(row) - }) + const stmt = db.prepare('SELECT * from tb_count') + const rows = stmt.all() + resolve(rows) }) } function setNum(name, num) { return new Promise((resolve, reject) => { - db.run(`INSERT INTO tb_count(\`name\`, \`num\`) + db.exec(`INSERT INTO tb_count(\`name\`, \`num\`) VALUES($name, $num) ON CONFLICT(name) DO UPDATE SET \`num\` = $num;` - , { - $name: name, - $num: num - } - , (err, row) => { - if (err) reject(err) + , + { $name: name, $num: num } + ) - resolve(row) - }) + resolve() + }) +} + +function setNumMulti(counters) { + return new Promise((resolve, reject) => { + const stmt = db.prepare(`INSERT INTO tb_count(\`name\`, \`num\`) + VALUES($name, $num) + ON CONFLICT(name) DO + UPDATE SET \`num\` = $num;`) + + const setMany = db.transaction((counters) => { + for (const counter of counters) stmt.run(counter) + }) + + setMany(counters) + resolve() }) } module.exports = { getNum, getAll, - setNum -} \ No newline at end of file + setNum, + setNumMulti +} diff --git a/index.js b/index.js index 88e6828..641348b 100644 --- a/index.js +++ b/index.js @@ -45,7 +45,7 @@ app.get('/get/@:name', async (req, res) => { const renderSvg = themify.getCountImage({ count: data.num, theme, length }) res.send(renderSvg) - console.log(data, `theme: ${theme}`) + console.log(data, `theme: ${theme}`, `ref: ${req.get('Referrer') || null}`, `ua: ${req.get('User-Agent') || null}`) }) // JSON record @@ -70,16 +70,51 @@ const listener = app.listen(config.app.port || 3000, () => { console.log('Your app is listening on port ' + listener.address().port) }) +let __cache_counter = {}, shouldPush = false + +setInterval(() => { + shouldPush = true +}, 1000 * 60); + +async function pushDB() { + if (!shouldPush) return + + try { + shouldPush = false + if (Object.keys(__cache_counter).length === 0) return + + console.log("pushDB", __cache_counter) + + const counters = Object.keys(__cache_counter).map(key => { + return { + name: key, + num: __cache_counter[key] + } + }) + + await db.setNumMulti(counters) + __cache_counter = {} + } catch (error) { + console.log("pushDB is error: ", error) + } +} + async function getCountByName(name) { const defaultCount = { name, num: 0 } if (name === 'demo') return { name, num: '0123456789' } try { - const counter = await db.getNum(name) || defaultCount - const num = counter.num + 1 - db.setNum(counter.name, num) - return counter + if (!(name in __cache_counter)) { + const counter = await db.getNum(name) || defaultCount + __cache_counter[name] = counter.num + 1 + } else { + __cache_counter[name]++ + } + + pushDB() + + return { name, num: __cache_counter[name] } } catch (error) { console.log("get count by name is error: ", error) diff --git a/package.json b/package.json index baf9850..df1b7d6 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "kawaii-counter", + "name": "moe-counter", "version": "1.0.0", "description": "", "main": "index.js", @@ -11,13 +11,13 @@ "author": "journey-ad", "license": "MIT", "dependencies": { + "better-sqlite3": "^8.1.0", "compression": "^1.7.4", "config-yml": "^0.10.3", "express": "^4.17.1", "image-size": "^0.8.3", "mime-types": "^2.1.27", "mongoose": "^5.9.28", - "pug": "^3.0.0", - "sqlite3": "^5.0.0" + "pug": "^3.0.0" } } diff --git a/views/index.pug b/views/index.pug index fa2dadf..4f681d3 100644 --- a/views/index.pug +++ b/views/index.pug @@ -62,7 +62,7 @@ html li a(href='https://repl.it/', target='_blank', rel='nofollow') repl.it li - a(href='https://www.asoulworld.com/', target='_blank', title='A-SOUL导航站(非官方)') A-SOUL + a(href='https://space.bilibili.com/703007996', target='_blank', title='A-SOUL_Official') A-SOUL li a(href='https://github.com/moebooru/moebooru', target='_blank', rel='nofollow') moebooru li @@ -109,4 +109,4 @@ html iframe(src="https://chat.getloli.com/room/@Moe-counter?title=%E8%90%8C%E8%90%8C%E8%AE%A1%E6%95%B0%E5%99%A8%E7%9A%84%E7%95%99%E8%A8%80%E6%9D%BF", scrolling="no", frameborder="0", height="70%", width="26%", style="position: fixed;top: 2%;right: 5%;") p.copy - a(href='https://github.com/journey-ad/Moe-counter', target='_blank', onclick='_evt_push("click", "normal", "go_github")') source code \ No newline at end of file + a(href='https://github.com/journey-ad/Moe-Counter', target='_blank', onclick='_evt_push("click", "normal", "go_github")') source code \ No newline at end of file