From 4831a91b9902770c7d2983513b7d5e3d74fee5cc Mon Sep 17 00:00:00 2001 From: Andrzej Stepien Date: Sat, 12 Aug 2023 17:41:48 +0200 Subject: [PATCH 1/8] transitioned from fetch to axios --- firefish-calls/createNote.mjs | 53 +++++++++----------------- firefish-calls/firefish.mjs | 12 ++++++ package-lock.json | 72 +++++++++++++++++++++++++++++++++++ package.json | 1 + 4 files changed, 104 insertions(+), 34 deletions(-) create mode 100644 firefish-calls/firefish.mjs diff --git a/firefish-calls/createNote.mjs b/firefish-calls/createNote.mjs index 1543c79..7951b54 100644 --- a/firefish-calls/createNote.mjs +++ b/firefish-calls/createNote.mjs @@ -1,44 +1,29 @@ -import { apiKey } from './API.mjs' +import firefish from './firefish.mjs' import logger from '../logger.mjs' -//JUST FOR USE IN LOCAL ENVIRONMENT -import { Agent, setGlobalDispatcher } from 'undici' -const agent = new Agent({ - connect: { - rejectUnauthorized: false - } -}) -setGlobalDispatcher(agent) -//^^^JUST FOR USE IN LOCAL ENVIRONMENT^^^ +// //JUST FOR USE IN LOCAL ENVIRONMENT +// import { Agent, setGlobalDispatcher } from 'undici' +// const agent = new Agent({ +// connect: { +// rejectUnauthorized: false +// } +// }) +// setGlobalDispatcher(agent) +// //^^^JUST FOR USE IN LOCAL ENVIRONMENT^^^ + export default async function createNote(text) { - const childLogger = logger.child({text}) - childLogger.trace("createNote called") - const url = 'http://localhost:3000/api/notes/create' - const params = { + logger.trace("createNote called") + const body = { text: text, } - const headers = { - "Authorization": "Bearer " + apiKey, - "Content-type": "application/json; charset=UTF-8" + try { + const response = await firefish.post("notes/create",body) + logger.info(response) + return response.data + } catch (error) { + throw error } - return await fetch(url, { - method: "POST", - headers: headers, - body: JSON.stringify(params), - - - }) - .then(res => { - return res.json() - }) - .then(data => { - childLogger.trace(data, "note created successfully") - return data - }) - .catch(error =>{ - throw error - }) } diff --git a/firefish-calls/firefish.mjs b/firefish-calls/firefish.mjs new file mode 100644 index 0000000..d6a41e4 --- /dev/null +++ b/firefish-calls/firefish.mjs @@ -0,0 +1,12 @@ +import axios from "axios" + +const apiKey = "sdJSmZpT518eOsNJUyiGNKGwjrcsQE1d" + +export default axios.create({ + baseURL: "http://localhost:3000/api/", + //timeout:1000, + headers: { + "Authorization": "Bearer " + apiKey, + "Content-type": "application/json; charset=UTF-8" + } +}) \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index a46f7d0..8ab272c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "GPL-3.0-or-later", "dependencies": { + "axios": "^1.4.0", "body-parser": "^1.20.2", "express": "^4.18.2", "node-cron": "^3.0.2", @@ -228,6 +229,11 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -236,6 +242,16 @@ "node": ">=8.0.0" } }, + "node_modules/axios": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz", + "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -416,6 +432,17 @@ "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz", "integrity": "sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==" }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/commander": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", @@ -483,6 +510,14 @@ "ms": "2.0.0" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -734,6 +769,38 @@ "node": ">= 0.8" } }, + "node_modules/follow-redirects": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -1867,6 +1934,11 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", diff --git a/package.json b/package.json index bf9015e..87876b8 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "sqlite3": "^5.1.6" }, "dependencies": { + "axios": "^1.4.0", "body-parser": "^1.20.2", "express": "^4.18.2", "node-cron": "^3.0.2", From 19c321c4590bfef1f462b481c16e8021bb200ec5 Mon Sep 17 00:00:00 2001 From: Andrzej Stepien Date: Sat, 12 Aug 2023 17:42:08 +0200 Subject: [PATCH 2/8] rationalised logging --- checkAndPublish.mjs | 3 +++ database-calls/insertPublished.mjs | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/checkAndPublish.mjs b/checkAndPublish.mjs index 94bb400..3955993 100644 --- a/checkAndPublish.mjs +++ b/checkAndPublish.mjs @@ -12,12 +12,15 @@ export default async function checkAndPublish () { if(!await todaysPromptAlreadyPublished()){ try { const prompt = await getNewPrompt() + logger.trace("getNewPrompt successful!") try { const text = makeText(prompt) try { const note = await createNote(text) + logger.trace("createNote successful!") try { await insertPublished(note, prompt.word) + logger.trace("insertPublished successful!") } catch (error) { logger.error(error, 'insertPublished failed!') } diff --git a/database-calls/insertPublished.mjs b/database-calls/insertPublished.mjs index b4f2e87..c4ad251 100644 --- a/database-calls/insertPublished.mjs +++ b/database-calls/insertPublished.mjs @@ -55,7 +55,6 @@ export default async function insertPublished(res, word) { date: isoDate(sampleRes.createdNote.createdAt) }) } catch (error) { - logger.trace("succesful insertion into table: published") throw error } From 0e04a8ed0440cb3b16c4dd13ad3275f97afba539 Mon Sep 17 00:00:00 2001 From: Andrzej Stepien Date: Sat, 12 Aug 2023 18:41:50 +0200 Subject: [PATCH 3/8] getting the buffer working --- checkAndPublish.mjs | 11 ++++--- database-calls/db.mjs | 73 ++++++++++++++++++++++++++++++++++++++----- getNewPrompt.mjs | 2 +- 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/checkAndPublish.mjs b/checkAndPublish.mjs index 3955993..8159361 100644 --- a/checkAndPublish.mjs +++ b/checkAndPublish.mjs @@ -3,22 +3,23 @@ import createNote from "./firefish-calls/createNote.mjs" import makeText from "./makeText.mjs" import insertPublished from "./database-calls/insertPublished.mjs" import logger from "./logger.mjs" -import {todaysPromptAlreadyPublished} from "./database-calls/db.mjs" +import {todaysPromptAlreadyPublished, tableIsNotEmpty, getPromptFromBuffer} from "./database-calls/db.mjs" export default async function checkAndPublish () { logger.trace("checkAndPublish called") logger.trace(todaysPromptAlreadyPublished()) if(!await todaysPromptAlreadyPublished()){ - try { - const prompt = await getNewPrompt() - logger.trace("getNewPrompt successful!") + try { + const prompt = await tableIsNotEmpty('buffer') ? await getPromptFromBuffer() : await getNewPrompt() + logger.trace("prompt acquired successfully!") try { const text = makeText(prompt) try { const note = await createNote(text) logger.trace("createNote successful!") try { + //DELETE FROM BUFFER await insertPublished(note, prompt.word) logger.trace("insertPublished successful!") } catch (error) { @@ -31,7 +32,7 @@ export default async function checkAndPublish () { logger.error(error, 'makeText failed!') } } catch (error) { - logger.error(error,'getNewPrompt failed!') + logger.error(error,'failed to get prompt!') } } else { logger.trace("today's prompt has already been published") diff --git a/database-calls/db.mjs b/database-calls/db.mjs index b01a092..80713ef 100644 --- a/database-calls/db.mjs +++ b/database-calls/db.mjs @@ -1,6 +1,7 @@ import Knex from 'knex' import logger from '../logger.mjs' import { isoDate } from "../utilities.mjs" +import config from "../config.mjs" export const db = Knex({ client: 'sqlite3', // or 'better-sqlite3' @@ -9,6 +10,36 @@ export const db = Knex({ }, useNullAsDefault: true }) +const blocklist = db.union([ + db('bad_words').select('word'), + db('medical_dictionary').select('word'), + db('published').select('word') +]) + +export const getAcceptablePrompts = async (word) => { + logger.trace("getAcceptablePrompt called") + try { + return db('dictionary') + .select('*') + .where({ + derivative: 0, + scientific: 0, + }) + .modify(queryBuilder=>{ + if(word){queryBuilder.andWhere('word',word)} + }) + .andWhere('count', '<', config.maxCount) + .andWhere('count', '>', config.minCount) + .andWhere('word', 'not in', blocklist) + .whereRaw('length(word) > 3') + .whereNotNull('pronunciation') + .orderByRaw('count desc') + } catch (error) { + logger.error("getAcceptablePrompts failed!") + throw error + } +} + export const getWords = async () => { logger.trace("getWords called") @@ -18,24 +49,52 @@ export const getWords = async () => { .catch(error => { throw error }) } -export const valueExistsInTable = async (table, column, value) => { + + + + +export const valueExistsInColumn = async (table, column, value) => { try { const number = await db(table) .count('* as count') .where(column, value) - return number[0].count > 0 + return number[0].count > 0 } catch (error) { - logger.error("valueExistsInTable failed!") + logger.error("valueExistsInColumn failed!") throw error } - + } export const todaysPromptAlreadyPublished = async () => { - return valueExistsInTable('published', 'date', isoDate()) + return valueExistsInColumn('published', 'date', isoDate()) } export const wordIsAlreadyInBuffer = async (word) => { - return valueExistsInTable('buffer', 'word', word) -} \ No newline at end of file + return valueExistsInColumn('buffer', 'word', word) +} + +export const tableIsNotEmpty = async (table) => { + try { + const number = await db(table) + .count('* as count') + return number[0].count > 0 + } catch (error) { + throw error + } +} + +export const getPromptFromBuffer = async () => { + logger.trace("getting prompt from buffer") + const oldestWordInBuffer = await db('buffer').select('word').orderBy('timestamp', 'asc').limit(1) + logger.info(`oldest word in buffer: ${oldestWordInBuffer[0].word}`) + try { + const prompt = await getAcceptablePrompts(oldestWordInBuffer[0].word) + return prompt[0] + } catch (error) { + logger.error("getPromptFromBuffer failed!") + throw error + } +} + diff --git a/getNewPrompt.mjs b/getNewPrompt.mjs index 85ffe34..f974d0f 100644 --- a/getNewPrompt.mjs +++ b/getNewPrompt.mjs @@ -1,5 +1,5 @@ import logger from "./logger.mjs" -import getAcceptablePrompts from "./database-calls/getAcceptablePrompts.mjs" +import { getAcceptablePrompts } from "./database-calls/db.mjs" import config from "./config.mjs" export default async function getNewPrompt() { From 4dcc369fbf9b5eb03732bea5961c0732549686ce Mon Sep 17 00:00:00 2001 From: Andrzej Stepien Date: Sat, 12 Aug 2023 18:42:05 +0200 Subject: [PATCH 4/8] should run on startup --- start.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/start.mjs b/start.mjs index 420e524..8c6a3ed 100644 --- a/start.mjs +++ b/start.mjs @@ -4,6 +4,7 @@ import checkAndPublish from './checkAndPublish.mjs'; export default function start(){ logger.trace("start() called") + checkAndPublish() cron.schedule('* * * * *', () => { checkAndPublish() logger.trace('one-minute cron call'); From 4a5882fcc2b2175d45d047838a25488bae4bf9a6 Mon Sep 17 00:00:00 2001 From: Andrzej Stepien Date: Sat, 12 Aug 2023 19:16:13 +0200 Subject: [PATCH 5/8] deleteFromBuffer now works --- checkAndPublish.mjs | 4 ++-- database-calls/db.mjs | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/checkAndPublish.mjs b/checkAndPublish.mjs index 8159361..ef39f4c 100644 --- a/checkAndPublish.mjs +++ b/checkAndPublish.mjs @@ -3,7 +3,7 @@ import createNote from "./firefish-calls/createNote.mjs" import makeText from "./makeText.mjs" import insertPublished from "./database-calls/insertPublished.mjs" import logger from "./logger.mjs" -import {todaysPromptAlreadyPublished, tableIsNotEmpty, getPromptFromBuffer} from "./database-calls/db.mjs" +import {deleteFromBuffer, todaysPromptAlreadyPublished, tableIsNotEmpty, getPromptFromBuffer} from "./database-calls/db.mjs" export default async function checkAndPublish () { @@ -19,7 +19,7 @@ export default async function checkAndPublish () { const note = await createNote(text) logger.trace("createNote successful!") try { - //DELETE FROM BUFFER + await deleteFromBuffer(prompt.word) await insertPublished(note, prompt.word) logger.trace("insertPublished successful!") } catch (error) { diff --git a/database-calls/db.mjs b/database-calls/db.mjs index 80713ef..a754cd8 100644 --- a/database-calls/db.mjs +++ b/database-calls/db.mjs @@ -98,3 +98,15 @@ export const getPromptFromBuffer = async () => { } } +export const deleteFromBuffer = async (word) => { + logger.trace(`deleteFromBuffer called for word ${word}!`) + try { + return db('buffer') + .where('word', word) + .del() + } catch (error) { + logger.error("deleteFromBuffer failed!") + } +} + +console.dir(await deleteFromBuffer('incapacity')) \ No newline at end of file From 8b31e868798f4bed67625e197e245b40fc854cf2 Mon Sep 17 00:00:00 2001 From: Andrzej Stepien Date: Sat, 12 Aug 2023 19:17:09 +0200 Subject: [PATCH 6/8] db calls refactor --- database-calls/getAcceptablePrompts.mjs | 30 ------------------------- social-interaction/receiveMention.mjs | 4 ++-- social-interaction/sendReply.mjs | 5 +++++ 3 files changed, 7 insertions(+), 32 deletions(-) delete mode 100644 database-calls/getAcceptablePrompts.mjs create mode 100644 social-interaction/sendReply.mjs diff --git a/database-calls/getAcceptablePrompts.mjs b/database-calls/getAcceptablePrompts.mjs deleted file mode 100644 index aaedf67..0000000 --- a/database-calls/getAcceptablePrompts.mjs +++ /dev/null @@ -1,30 +0,0 @@ -import logger from "../logger.mjs" -import { db } from "./db.mjs" -import config from "../config.mjs" - -const blocklist = db.union([ - db('bad_words').select('word'), - db('medical_dictionary').select('word'), - db('published').select('word') -]) - -export default async () => { - logger.trace("getAcceptablePrompt called") - try { - return db('dictionary') - .select('*') - .where({ - derivative: 0, - scientific: 0, - }) - .andWhere('count', '<', config.maxCount) - .andWhere('count', '>', config.minCount) - .andWhere('word', 'not in', blocklist) - .whereRaw('length(word) > 3') - .whereNotNull('pronunciation') - .orderByRaw('count desc') - } catch (error) { - logger.error("getAcceptablePrompts failed!") - throw error - } -} \ No newline at end of file diff --git a/social-interaction/receiveMention.mjs b/social-interaction/receiveMention.mjs index 945ae93..93e247d 100644 --- a/social-interaction/receiveMention.mjs +++ b/social-interaction/receiveMention.mjs @@ -1,6 +1,6 @@ import logger from "../logger.mjs" -import getAcceptablePrompts from "../database-calls/getAcceptablePrompts.mjs" import { checkSpelling } from "spellchecker" +import { wordIsAlreadyInBuffer, getAcceptablePrompts } from "../database-calls/db.mjs" export default async function (note) { const childLogger = logger.child({note}) @@ -13,7 +13,7 @@ export default async function (note) { const word = textArray[0] if(checkSpelling(word)){ if(await getAcceptablePrompts().indexOf(word)!=-1){ - if(/*wordIsAlreadyInBuffer*/true){ + if(!wordIsAlreadyInBuffer(word)){ } } diff --git a/social-interaction/sendReply.mjs b/social-interaction/sendReply.mjs new file mode 100644 index 0000000..8ff310c --- /dev/null +++ b/social-interaction/sendReply.mjs @@ -0,0 +1,5 @@ + + +export default async () => { + +} \ No newline at end of file From a7c66c86ecbf93928eede2b433727c1d0c6a8d60 Mon Sep 17 00:00:00 2001 From: Andrzej Stepien Date: Sun, 13 Aug 2023 13:00:12 +0200 Subject: [PATCH 7/8] factored out, moved into db.mjs --- database-calls/db.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/database-calls/db.mjs b/database-calls/db.mjs index a754cd8..5b6e3e8 100644 --- a/database-calls/db.mjs +++ b/database-calls/db.mjs @@ -109,4 +109,3 @@ export const deleteFromBuffer = async (word) => { } } -console.dir(await deleteFromBuffer('incapacity')) \ No newline at end of file From e54767c09cbcb167ceb7d819769612d70981316c Mon Sep 17 00:00:00 2001 From: Andrzej Stepien Date: Sun, 13 Aug 2023 13:00:24 +0200 Subject: [PATCH 8/8] factored out, moved to db.mjs --- database-calls/todaysPromptAlreadyPublished.mjs | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 database-calls/todaysPromptAlreadyPublished.mjs diff --git a/database-calls/todaysPromptAlreadyPublished.mjs b/database-calls/todaysPromptAlreadyPublished.mjs deleted file mode 100644 index d1942e1..0000000 --- a/database-calls/todaysPromptAlreadyPublished.mjs +++ /dev/null @@ -1,17 +0,0 @@ -import { db } from "./db.mjs"; -import { isoDate } from "../utilities.mjs"; -import logger from "../logger.mjs"; -export default async function todaysPromptAlreadyPublished() { - try { - const number = await db('published') - .count('* as count') - .where('date', isoDate()) - return number[0].count > 0 - } catch (error) { - logger.error("todaysPromptAlreadyPublished failed!") - throw error - } - -} - -//console.log(await todaysPromptAlreadyPublished())