From 3d434a570ca6348ae3d04f01079a20c4e369c7c7 Mon Sep 17 00:00:00 2001 From: Andrzej Stepien Date: Mon, 7 Aug 2023 00:24:49 +0200 Subject: [PATCH] getNewPrompt function --- src/data/database | Bin 35254272 -> 35266560 bytes src/getNewPrompt.mjs | 41 +++++++++++++++++++++++++++++++++++++++++ src/skewNormal.mjs | 29 +++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 src/getNewPrompt.mjs create mode 100644 src/skewNormal.mjs diff --git a/src/data/database b/src/data/database index 61d2faea7a7bc02c32fd4a304a3cd5056edc6f47..4be6a8777d2a3e0e1ae39b164bdcc645dc57b669 100644 GIT binary patch delta 2823 zcmbu>cT`tp9Ds57fgs3qVVMD0IWt8oZQ>x!%+e^$GKBc4B!UCdhKgG5fo2v~rk0vE zoN2Cbv{ReTmbuH8wo9$&@pSZbPW{zA=lR_4x%a;BfA^kSvgc4j$@;?yr8N_mg@mNe z2|07bPHEV**0PXBUE0US?GA~G*%%UcJbqJLX>3_sPF%B?jj_SCr3GYgCU2 z4C@;inUtIy7+$9+J10NbuQ(?^Z(#3W?&%?QR_o3gEi*GxGF!Ilkdbowdfn(jg#&U^ zih_fSQVI&GI6pt7xFD~8aj;=@Zgx@dUvJe*i;76fNDhSL732m75A2`cHLoa`m0es^ zcFu-9PLHW`(klLQ(SNOJa%Ro!k!7)?%VI+*Kw%V46)1utsUk&DG*zM)a!nk? zQvxMYWvW6|sTx(MB&tDY(b-g!&Y@a#E+tb6rP6s+o6e^T=t8PPb*UcJrv}uJE}}+s zF*T+pltxXd8C^n`(q+_~TF~Wm1zky3(baSfrBh34MXjj~WzeO`ICX3C^n=vL}NUFkOJMz_-))Sd36Eb2koltXt>E(NJ4^`hRCM|V>n%BQ|m zK!wzg`qKa!NJUgk_s}33O!v}#bU!^nLue=sqY@fUBWNT&NDt8{8ckzpERCb_G=V14 zB$`a6G=-+p!!(T^q3QG}Jw|0TgC3_R=t-JMPth!zO>^jJdWPoGJep6>(gIpY&(ZVr z0=-C!XfZ9JrL>He(@V61R?^F~idNGaT1)Hb6)LCow1GC#CVG`#qu1#TdXwIwx9J_) zOj~Fxy-V-W`}6^QNFUKQ+D0MYo)EU zlk23tTrVBu2I(j_%1zQqI?K(HDYwY2(nY$;ZPHC{mpi1p+$mYoL$W1D?vh*yN>Aw} zy(LfXmOhd%eWgGOrJwYd0WwgEq*(5eK{8nGmHXs=c|eB9P#GpAGF(Q;NO@2ml2I~R z#>iM1C*x&;Oq5A7SxRM!OqGXanmi)YFmlGE<(CSu$JZ$kXzS%$0dE zU!Ii(vQVCr=j8=?Q5MN!St3hinJkx=WQDAhmt~c#mNl|g*2ybUF6(83Y?Mv%s=Owz z%Nz2hyd`hTJF;1}$X0n*-jny`1Nl%sl5MhGcF0cICA;Ng`9waI&*XF2BVWi~*(dwu zfE<)Va#+5UujGh)E#Jtu@|}DyN9CCOAV11a^0WLRzshg&yZj-?<%FD+KSTNa1sFml zAYl?N6(m9;rJ_Vhv{aH9IkiEYQ@kWdqEwbDQdO!+bxD#MfpR{7fnIwn#QIZOo;Yh- fXjxc!<2{Q*XN3QKTA5Ok*(hv!Y2)zF|MkFMmhK32 delta 2510 zcmYM!Rg@4!7(n3-SaOS8T9EFNP*OUiOQb`(!2s#*kXk^xn^gg+r5gdIrMr<3MNqt# z7w4S&o&RO#ZT>+!??w!|dM9F1n#h=dfQ<73{y&K)=Pnc#6OgxI)zGjD0U@E$0by^# z_k>LfjSg!S78_ZtUSNUX*!&UUL9;_*%d}q+*dTalbm++F&_D{JV2VR=DTLxtd`dtG zDG?2mSNQI~{eLzL1C>5jP^dWsjC8#8oqS91`%F@SFj>=O7sz{ZnGF73f zRE?_BCsc!KQZ1@Yb*L`Yqx#f<8d4+rlp51#)P$N+GipvPs3oPfw*H}#<}=u7&FzNWs^kG`S)G=K)uAR0_VXebS%;WUCq(kL2D zV`wamqwzF>CekFDOjGDvno84XI?bS&6iu^eHqD{$XfDm8`Luu*(jr<+OK2%Aqvf=M zR?;e3O>1Z^t)um{fnw-;`hk9=jkJk2(@(U8w$e7*PCF=;cG51|O?zlB?W6s4fDY0j zI!s6CC>^7p={TLBlk^MyN~h>FouRXIj?U8s`i(BqCAv&k=qg>K>vV%|(k;48cjzwN zqxjeIlZ7i=q3G0f6*)Yn_kl!`iK6dw`2MH6G?(3SmH=r36Xdb zUlK?{NhFEo-2-7x;SwQ{l0=e9GD$8eB&9@2DoHJAB(0>A^pZi|lZ=u{-j~dhMY2jZ z$u2o0r{t2{l1K7NKFKcyq@Wa%!t#L>k)l#eipz)ck(7{WV`H;SlKDNWVh^*y|PdC z%K7CO|Hugxhc2g zw%n1sa!>Bd19>QqMG^i6`+Tfh3efl33n75atvv5fUj$B&j5morR>o2I?_5T107MQaD diff --git a/src/getNewPrompt.mjs b/src/getNewPrompt.mjs new file mode 100644 index 0000000..32f308d --- /dev/null +++ b/src/getNewPrompt.mjs @@ -0,0 +1,41 @@ +import { db } from "./db.mjs" +import { randomSkewNormal, randomSkewNormalTrimmed } from "./skewNormal.mjs" + +const maxCount = 30000000 +const minCount = 200000 + +export default async function getNewPrompt() { + const badWords = await db('bad_words') + .select('word') + + const prompts = await db('dictionary') + .select('*') + .where({ + derivative: 0, + scientific: 0, + }) + .andWhere('count', '<', maxCount) + .andWhere('count', '>', minCount) + .andWhere('word', 'not in', badWords) + .whereNotNull('pronunciation') + .orderBy('count') + + + const randomEntry = (array) => { + //const random = (randomSkewNormal(Math.random,0,2,0)/6) + const random = Math.random() + return array[ + parseInt( + array.length * random + ) + ] + } + db.destroy() + return randomEntry(prompts) +} + + + +console.log(await getNewPrompt()) + + diff --git a/src/skewNormal.mjs b/src/skewNormal.mjs new file mode 100644 index 0000000..6b23acb --- /dev/null +++ b/src/skewNormal.mjs @@ -0,0 +1,29 @@ +const randomNormals = (rng) => { + let u1 = 0, u2 = 0; + //Convert [0,1) to (0,1) + while (u1 === 0) u1 = rng(); + while (u2 === 0) u2 = rng(); + const R = Math.sqrt(-2.0 * Math.log(u1)); + const Θ = 2.0 * Math.PI * u2; + return [R * Math.cos(Θ), R * Math.sin(Θ)]; +} + +export const randomSkewNormal = (rng, ξ, ω, α = 0) => { + const [u0, v] = randomNormals(rng); + if (α === 0) { + return ξ + ω * u0; + } + const 𝛿 = α / Math.sqrt(1 + α * α); + const u1 = 𝛿 * u0 + Math.sqrt(1 - 𝛿 * 𝛿) * v; + const z = u0 >= 0 ? u1 : -u1; + return ξ + ω * z; +} + +export const randomSkewNormalTrimmed = ()=>{ + let random = 1 + do { + random = randomSkewNormal(Math.random,0,2,0) + } while (random<0 || random>0.999999) + return random +} +