Skip to content
Snippets Groups Projects
Commit 7f94cc25 authored by Millicent Billette's avatar Millicent Billette
Browse files

WiP: dictionary conversion to new engine.

parent 199d4bcb
No related branches found
No related tags found
No related merge requests found
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 1938 additions and 783 deletions
CI/minify.mjs
+ 2
− 13
View file @ 7f94cc25
import {readdirSync, readFileSync, writeFileSync} from 'node:fs'; import {readdirSync, readFileSync, writeFileSync} from 'node:fs';
import {minify} from 'terser'; import {transform} from 'esbuild';
const minifyCache = {};
minFold('generated/npm/browser/'); minFold('generated/npm/browser/');
minFold('generated/npm/nodejs/'); minFold('generated/npm/nodejs/');
...@@ -9,17 +8,7 @@ async function minFold(folder) { ...@@ -9,17 +8,7 @@ async function minFold(folder) {
readdirSync(folder).forEach(async fileName => { readdirSync(folder).forEach(async fileName => {
if (!fileName.includes('.mjs')) return; if (!fileName.includes('.mjs')) return;
const orgContent = readFileSync(folder + fileName, 'utf8').replace(/import '[^ ']+';/g,''); const orgContent = readFileSync(folder + fileName, 'utf8').replace(/import '[^ ']+';/g,'');
const minified = await minify(orgContent, { const minified = await transform(orgContent, { minify: true, treeShaking:true, target: "esnext", legalComments:'none', format: 'esm' });
toplevel: true,
ecma: 2020,
nameCache: minifyCache,
compress: {
passes: 2
},
format: {
comments: false
}
});
writeFileSync(folder + fileName, minified.code); writeFileSync(folder + fileName, minified.code);
}); });
} }
package-lock.json
+ 1485
− 516
View file @ 7f94cc25
This diff is collapsed.
package.json
+ 5
− 5
View file @ 7f94cc25
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
"build:npm:rollup": "rollup --config CI/rollup.config.mjs", "build:npm:rollup": "rollup --config CI/rollup.config.mjs",
"build:npm:cp": "cp npm/* generated/npm/", "build:npm:cp": "cp npm/* generated/npm/",
"build:npm:cp:readme": "cp README* generated/npm/", "build:npm:cp:readme": "cp README* generated/npm/",
"build:npm:min:terser": "node CI/minify.mjs", "build:npm:min": "node CI/minify.mjs",
"test": "run-s test:dev", "test": "run-s test:dev",
"test:dev": "run-s test:dev:**", "test:dev": "run-s test:dev:**",
"xtest:dev:qualityCheck": "xo", "xtest:dev:qualityCheck": "xo",
...@@ -55,15 +55,15 @@ ...@@ -55,15 +55,15 @@
"c8": "^7.12.0", "c8": "^7.12.0",
"chokidar-cli": "^3.0.0", "chokidar-cli": "^3.0.0",
"es6-plato": "https://github.com/1000i100/es6-plato#master", "es6-plato": "https://github.com/1000i100/es6-plato#master",
"esbuild": "^0.15.15",
"eslint-plugin-ava": "^13.2.0", "eslint-plugin-ava": "^13.2.0",
"eslint-plugin-promise": "^6.1.1", "eslint-plugin-promise": "^6.1.1",
"eslint-plugin-unicorn": "^44.0.2", "eslint-plugin-unicorn": "^45.0.0",
"jscpd": "^3.5.1", "jscpd": "^3.5.1",
"mkdirp": "^1.0.4", "mkdirp": "^1.0.4",
"npm-run-all": "^4.1.5", "npm-run-all": "^4.1.5",
"rollup": "^3.3.0", "rollup": "^3.4.0",
"terser": "^5.15.1", "xo": "^0.53.1"
"xo": "^0.52.4"
}, },
"disabledDependenciesTODOAddComplexityQualityCheck": { "disabledDependenciesTODOAddComplexityQualityCheck": {
"ecma-nacl": "^2.5.0", "ecma-nacl": "^2.5.0",
......
src/crypto.mjs
+ 1
− 4
View file @ 7f94cc25
...@@ -207,9 +207,7 @@ export function strOrBin2bin(strOrBin){ ...@@ -207,9 +207,7 @@ export function strOrBin2bin(strOrBin){
const encoder = new TextEncoder(); const encoder = new TextEncoder();
return (typeof strOrBin === 'string')?encoder.encode(strOrBin):strOrBin; return (typeof strOrBin === 'string')?encoder.encode(strOrBin):strOrBin;
} }
export function b64orBin2bin(b64orBin){
return typedStrOrBin2Bin(b64orBin,b64.decode);
}
export function b58orBin2bin(b58orBin){ export function b58orBin2bin(b58orBin){
return typedStrOrBin2Bin(b58orBin,b58.decode); return typedStrOrBin2Bin(b58orBin,b58.decode);
} }
...@@ -224,7 +222,6 @@ function secKey2bin(secretKey){ ...@@ -224,7 +222,6 @@ function secKey2bin(secretKey){
export function textEncrypt(jsonMessage, senderPrivateKey, receiverPubKey, nowInSecond=0, b58nonce=''){ export function textEncrypt(jsonMessage, senderPrivateKey, receiverPubKey, nowInSecond=0, b58nonce=''){
const bNonce = b58nonce?b58.decode(b58nonce):nacl.randomBytes(NONCE_BYTES); const bNonce = b58nonce?b58.decode(b58nonce):nacl.randomBytes(NONCE_BYTES);
console.log("Défini : ", receiverPubKey)
const bReceiverPubKey = convertPublicKey(pubKey2bin(receiverPubKey)); const bReceiverPubKey = convertPublicKey(pubKey2bin(receiverPubKey));
const bSenderPrivateKey = convertSecretKey(secKey2bin(senderPrivateKey)); const bSenderPrivateKey = convertSecretKey(secKey2bin(senderPrivateKey));
const encrypt = utf8text => b64.encode(nacl.box(strOrBin2bin(utf8text),bNonce,bReceiverPubKey,bSenderPrivateKey)); const encrypt = utf8text => b64.encode(nacl.box(strOrBin2bin(utf8text),bNonce,bReceiverPubKey,bSenderPrivateKey));
......
src/dictionary-builder.mjs deleted 100644 → 0
+ 0
− 157
View file @ 199d4bcb
import latinize from '../node_modules/latinize-to-ascii/latinize.mjs';
export {dedup, noAccentVariant, casesVariants, regLikeVariants, resetCache};
function dedup(array) {
const result = {};
array.forEach(v => result[v] = v); // eslint-disable-line no-return-assign
return Object.keys(result);
}
function noAccentVariant(string) {
return dedup([string, latinize(string)]);
}
function casesVariants(string) {
return dedup([
string,
string.toLowerCase(),
string.split(' ').map(str => str.charAt(0).toUpperCase() + str.slice(1).toLowerCase()).join(' '),
string.toUpperCase()
]);
}
function regLikeVariants(theString, allStrings = []) {
return _regLikeVariants(escape2utfSpecial(theString), allStrings.map(escape2utfSpecial)).map(utfSpecial2unEscaped);
}
function swapKeyValue(object) {
const result = {};
for (const key in object) {
result[object[key]] = key;
}
return result;
}
const specialMap = {
'(': String.fromCharCode(0x01),
')': String.fromCharCode(0x02),
'|': String.fromCharCode(0x03),
'{': String.fromCharCode(0x04),
'}': String.fromCharCode(0x05),
',': String.fromCharCode(0x06),
'[': String.fromCharCode(0x07),
']': String.fromCharCode(0x08),
'-': String.fromCharCode(0x09),
'<': String.fromCharCode(0x0a),
'>': String.fromCharCode(0x0b),
':': String.fromCharCode(0x0c),
'=': String.fromCharCode(0x0d)
};
const revertSpecial = swapKeyValue(specialMap);
function escape2utfSpecial(str) {
return str.replace(/\\(.)/g, (a, chr) => specialMap[chr] ? specialMap[chr] : chr);
}
function utfSpecial2unEscaped(str) {
return str.split('').map(chr => revertSpecial[chr] ? revertSpecial[chr] : chr).join('');
}
function bracketsHandler(theString) {
// Handle []
const lower = 'abcdefghijklmnopqrstuvwxyz';
const upper = lower.toUpperCase();
const number = '0123456789';
theString = theString.replace(/(\[[^\]]*)([a-z]-[a-z])([^\]]*])/g, (osef, before, chrs, after) => before + lower.slice(lower.indexOf(chrs.split('-')[0]), lower.indexOf(chrs.split('-')[1]) + 1) + after);
theString = theString.replace(/(\[[^\]]*)([A-Z]-[A-Z])([^\]]*])/g, (osef, before, chrs, after) => before + upper.slice(upper.indexOf(chrs.split('-')[0]), upper.indexOf(chrs.split('-')[1]) + 1) + after);
theString = theString.replace(/(\[[^\]]*)([0-9]-[0-9])([^\]]*])/g, (osef, before, chrs, after) => before + number.slice(number.indexOf(chrs.split('-')[0]), number.indexOf(chrs.split('-')[1]) + 1) + after); // eslint-disable-line unicorn/better-regex
theString = theString.replace(/\[([^\]]+)]/g, (osef, chrs) => `(${chrs.split('').join('|')})`);
return theString;
}
function resetCache() {
cache();
}
function cache(key, func, ...args) {
/* Init */ if (!cache.cached) cache.cached = {};
/* Reset cache */ if (arguments.length === 0) return cache.cached = {}; // eslint-disable-line no-return-assign
/* Compute and cache */ if (typeof cache.cached[key] === 'undefined') cache.cached[key] = func(...args);
/* Answer from cache */ return cache.cached[key];
}
function labelExpressions(str) {
return str.slice(str.indexOf('::') + 2);
}
function computedLabel(label, allStrings) {
return cache(`label::${label}`, _computedLabel, label, allStrings);
}
function _computedLabel(label, allStrings) {
const matchingLabel = allStrings.filter(str => str.trim().indexOf(`${label}::`) === 0);
return flatten(matchingLabel.map(str => _regLikeVariants(labelExpressions(str)), allStrings));
}
function refHandler(theString, allStrings) {
// Handle <ref>
theString = theString.replace(/<([^>]+)>/g, (osef, ref) => `(${computedLabel(ref, allStrings).join('|')})`);
return theString;
}
function syncRefHandler(theString, allStrings) {
// Handle =ref>
const syncRef = dedup(theString.match(/=[^>]+>/g) || []).map(str => str.substring(1, str.length - 1));
let results = [theString];
for (const ref of syncRef) {
const variantes = computedLabel(ref, allStrings);
const tmpRes = [];
for (const v of variantes) {
for (const r of results) {
tmpRes.push(r.replace(new RegExp(`=${ref}>`, 'g'), v));
}
}
results = tmpRes;
}
return results;
}
function qtyHandlerReplaceCallback(all, chr, qty) {
const mm = qty.split(',').map(n => n.trim() * 1); // eslint-disable-line no-implicit-coercion
const min = mm[0];
const max = (mm.length === 2) ? mm[1] : min;
const result = [];
for (let i = min; i <= max; i++) result.push(new Array(i + 1).join(chr)); // eslint-disable-line unicorn/no-new-array
return `(${result.join('|')})`;
}
function qtyHandler(theString) {
// Handle {qty} and {min,max}
theString = theString.replace(/([^)]){([^}]+)}/g, qtyHandlerReplaceCallback);
theString = theString.replace(/^(.*)\(([^)]*)\){([^}]+)}(.*)$/, (all, before, choices, qty, after) => before + qtyHandlerReplaceCallback('', `(${choices})`, qty) + after); // eslint-disable-line max-params
return theString;
}
function optionsHandler(theString, allStrings) {
// Handle (|)
let multiString = theString.replace(/^(.*)\(([^)]*)\)(.*)$/, (all, before, choices, after) => choices.split('|').map(c => before + c + after).join('=$##$=')).split('=$##$=');
multiString = [].concat(...multiString.map(str => (str.includes('(')) ? _regLikeVariants(str, allStrings) : str));
return dedup(multiString);
}
function _regLikeVariants(theString, allStrings) {
if (theString.indexOf('::') > 0) return []; // Remove label definition lines.
theString = bracketsHandler(theString); // Handle []
theString = refHandler(theString, allStrings);// Handle <ref> and ref::
theString = qtyHandler(theString); // Handle {qty} and {min,max}
const strings = optionsHandler(theString, allStrings); // Handle (|)
return flatten(strings.map(string => syncRefHandler(string, allStrings)));// Handle =ref> and ref::
}
function flatten(arrayOfArray) {
return dedup([].concat(...arrayOfArray));
}
src/dictionary-builder.test.mjs deleted 100644 → 0
+ 0
− 69
View file @ 199d4bcb
import test from 'ava';
import * as app from './dictionary-builder.mjs';
test.beforeEach(app.resetCache);
test('add no accents variant', t => {
t.deepEqual(app.noAccentVariant('Ǧ1'), ['Ǧ1', 'G1']);
});
test('add case variants', t => {
t.deepEqual(app.casesVariants('moT'), ['moT', 'mot', 'Mot', 'MOT']);
});
test('add multi word case variants', t => {
t.deepEqual(app.casesVariants('autre mot'), ['autre mot', 'Autre Mot', 'AUTRE MOT']);
});
test('regLikeVariants remove ref:: lines', t => {
t.deepEqual(app.regLikeVariants('ref::truc'), []);
});
// TODO: handle ref infinite loop by returning §infiniteRecursion§
test('regLikeVariants handle <ref>', t => {
t.deepEqual(app.regLikeVariants('<ref> <ref>', ['ref::truc', 'ref::bidule', '<ref> <ref>']), ['truc truc', 'bidule truc', 'truc bidule', 'bidule bidule']);
t.deepEqual(app.regLikeVariants('<ref> <ref>', ['ref::(truc|bidule)', '<ref> <ref>']), ['truc truc', 'bidule truc', 'truc bidule', 'bidule bidule']);
});
test('regLikeVariants handle =ref>', t => {
t.deepEqual(app.regLikeVariants('=ref> =ref>', ['ref::truc', 'ref::bidule', '=ref> =ref>']), ['truc truc', 'bidule bidule']);
t.deepEqual(app.regLikeVariants('=ref> =ref>', ['ref::(truc|bidule)', '=ref> =ref>']), ['truc truc', 'bidule bidule']);
});
test('regLikeVariants handle multiple =ref>', t => {
t.deepEqual(
app.regLikeVariants('=ref> =ref2> =ref> =ref2>', ['ref::(truc|bidule)', 'ref2::(machin|chose)', '=ref> =ref>']),
['truc machin truc machin', 'bidule machin bidule machin', 'truc chose truc chose', 'bidule chose bidule chose']);
});
test('regLikeVariants handle (this|that)', t => {
t.deepEqual(app.regLikeVariants('(this|that)'), ['this', 'that']);
});
test('regLikeVariants handle [ -_]', t => {
t.deepEqual(app.regLikeVariants('[ -_]'), [' ', '-', '_']);
});
test('regLikeVariants handle [a-f]', t => {
t.deepEqual(app.regLikeVariants('[a-f]'), ['a', 'b', 'c', 'd', 'e', 'f']);
t.deepEqual(app.regLikeVariants('[7-9]'), ['7', '8', '9']);
t.deepEqual(app.regLikeVariants('[C-F]'), ['C', 'D', 'E', 'F']);
});
test('regLikeVariants handle [a-c-]', t => {
t.deepEqual(app.regLikeVariants('[a-c-]'), ['a', 'b', 'c', '-']);
});
test('regLikeVariants handle {qty}', t => {
t.deepEqual(app.regLikeVariants('a{5}'), ['aaaaa']);
});
test('regLikeVariants handle {min,max}', t => {
t.deepEqual(app.regLikeVariants('b{3,5}'), ['bbb', 'bbbb', 'bbbbb']);
});
test('regLikeVariants handle (string){qty}', t => {
t.deepEqual(app.regLikeVariants(`c'est (toto|tata){0,2}`), [`c'est `, `c'est toto`, `c'est totototo`, `c'est tata`, `c'est tatatoto`, `c'est tototata`, `c'est tatatata`]);
});
test.serial('regLikeVariants handle nested -([a-f]|<ref>){0,1}', t => {
t.deepEqual(
app.regLikeVariants('-([B-D]|<ref>){0,1}', ['ref::plop']),
['-', '-B', '-plop', '-C', '-D']);
});
test('regLikeVariants handle plop:\\:', t => {
t.deepEqual(app.regLikeVariants('plop:\\:ici'), ['plop::ici']);
t.deepEqual(app.regLikeVariants('plop\\::ici'), ['plop::ici']);
t.deepEqual(app.regLikeVariants('plop::ici'), []);
});
test('regLikeVariants handle [\\]*]', t => {
t.deepEqual(app.regLikeVariants('[\\]*]'), [']', '*']);
});
test('regLikeVariants handle escaping common chr \\a', t => {
t.deepEqual(app.regLikeVariants('\\a'), ['a']);
});
src/dictionary-parser.mjs
+ 289
− 10
View file @ 7f94cc25
import latinize from '../node_modules/latinize-to-ascii/latinize.mjs';
import * as tree from './dictionary-tree.mjs'; import * as tree from './dictionary-tree.mjs';
export function parse(dictionaryString) { export function parse(dictionaryString,idSecPwd=true,accent=0,lowerCase=0,leet=0) {
const allLines = dictionaryString.split('\n'); resetCache();
const escapedString = escape2utfSpecial(dictionaryString);
const unbracketed = bracketsHandler(escapedString);
const allLines = unbracketed.split('\n');
const parsedLines = parseAllLines(allLines); const parsedLines = parseAllLines(allLines);
return combineUnspecified2IdSecPwd(parsedLines); const monoLined = `(${parsedLines.join('|')})`
.replace(/§void§\|/g,'')
.replace(/\|§void§/g,'')
.replace(/§void§/g,'');
if(monoLined.includes('§infiniteRecursion§')) throw new Error(`Unable to parse : ${utfSpecial2unEscaped(flattenIt(monoLined))}`);
if(!idSecPwd) return parseEnd(monoLined,allLines,accent,lowerCase,leet);
const parts = tree.splitAround('@@',tree.buildTreeStruct(monoLined));
const flatParts = [];
if(parts.matching) flatParts.push(parts.matching);
if(parts.notMatching) flatParts.push(parts.notMatching);
const combined = combineUnspecified2IdSecPwd(flatParts);
return parseEnd(combined,allLines,accent,lowerCase,leet);
}
function parseEnd(str,allLines,accent,lowerCase,leet) {
let theString = syncRefHandler(str,allLines)
if(theString.match(/=[^>]+>/)) throw new Error(`Unable to parse : ${utfSpecial2unEscaped(flattenIt(theString))}`);
theString = qtyHandler(theString);
if(accent===1) theString = zeroAccent(theString);
if(accent===2) theString = optionalAccent(theString);
if(lowerCase===1) theString = zeroUpperCase(theString);
if(lowerCase===2) theString = optionalUpperCase(theString);
if(lowerCase===3) theString = allUpperCase(allCapitalized(theString));
if(lowerCase===4) theString = optionalCapitalized(theString);
if(leet===1) theString = optionalLeetSpeak(theString);
if(leet===2) theString = allLeetSpeak(theString);
if(leet===3) theString = everyLeetSpeak(theString);
//TODO: bigLeet Variants
return utfSpecial2unEscaped(flattenIt(theString));
}
function zeroVariant(str,func){
const zero = func(str);
if(zero===str) return str;
return `(${str}|${zero})`;
}
function optionalVariant(str,func) {
return str.split('').map(func).join('');
}
function zeroAccent(string) {
return zeroVariant(string,latinize);
}
function optionalAccent(string) {
return optionalVariant(string,zeroAccent);
}
function zeroUpperCase(string) {
return zeroVariant(string,str=>str.toLowerCase() );
}
function allUpperCase(string) {
return zeroVariant(string,str=>str.toUpperCase() );
}
function optionalUpperCase(string) {
return optionalVariant(string,zeroUpperCase);
}
function allCapitalized(string) {
let str = zeroVariant(flattenIt(string),str=>str.split(' ').map(s => s.charAt(0).toUpperCase() + s.slice(1)).join(' ') );
let str2 = zeroVariant(flattenIt(string),str=>str.split(' ').map(s => s.charAt(0).toUpperCase() + s.slice(1).toLowerCase()).join(' ') );
return zeroVariant(str,()=>str2 );
}
function optionalCapitalized(string) {
return optionalUpperCase(allCapitalized(string));
}
function zeroLeetSpeak(string) {
return zeroVariant(string,fromLeet);
}
function allLeetSpeak(string) {
return zeroVariant(string,toLeet);
}
function optionalLeetSpeak(string) {
return optionalVariant(string,zeroLeetSpeak);
}
function everyLeetSpeak(string) {
return optionalLeetSpeak(optionalVariant(string,allLeetSpeak));
}
function replaceList(applyOnThisString,list){
return applyOnThisString.split('').map(chr => list[chr] || chr).join('');
}
function replaceUpperKeyList(applyOnThisString,list){
return applyOnThisString.split('').map(chr => list[chr.toUpperCase()] || chr).join('');
}
/*
function toBigLeet(str){
return replaceUpperKeyList(str,{
'A': '(4|@|^|∂|q)',
'B': '(8|6|13|ß|!3|/3)',
'C': '(¢|©)',
'D': '(0|?)',
'E': '(3|&|€|£|є|ë)',
'F': '(ƒ|φ)',
'G': '(6|&|9)',
'H': '(#|?)',
'I': '(1|!)',
'J': '(;|¿)',
'K': '(x|X)',
'L': '(1|£)',
'M': '^^',
'N': '/V',
'O': '(0|°|¤)',
'P': '(9|7|?|φ)',
'Q': '0',
'R': '(2|z|Z|Я|®)',
'S': '(5|$|z|§)',
'T': '(7|1|†|+)',
'U': '(v|V|µ)',
'V': '(u|U|µ)',
'W': 'vv',
'X': '(8|×)',
'Y': '(7|j|λ)',
'Z': '(2|%)',
});
}
function fromBigLeet(str) {
//TODO:fromBigLeet
}
*/
function toLeet(str){
return replaceUpperKeyList(str,{
'O': '0',
'I': '1',
'L': '1',
'R': '2',
'Z': '2',
'E': '3',
'A': '(4|@)',
'S': '(5|$)',
'G': '(6|9)',
'T': '(7|1)',
'Y': '7',
'B': '(8|6)',
});
}
function fromLeet(str){
return replaceList(str,{
'@': '(a|A)',
'$': '(s|S)',
'0': '(o|O)',
'1': '(i|l|I|L|t|T)',
'2': '(r|R|z|Z)',
'3': '(e|E)',
'4': '(a|A)',
'5': '(s|S)',
'6': '(g|G|b|B)',
'7': '(t|T|y|Y)',
'8': '(b|B)',
'9': '(g|G)',
});
}
function flattenIt(regStr){
return tree.serialize(tree.buildTreeStruct(regStr));
}
function resetCache() {
cache();
}
function cache(key, func, ...args) {
/* Init */ if (!cache.cached) cache.cached = {};
/* Reset cache */ if (arguments.length === 0) return cache.cached = {}; // eslint-disable-line no-return-assign
/* Compute and cache */ if (typeof cache.cached[key] === 'undefined') cache.cached[key] = func(...args);
/* Answer from cache */ return cache.cached[key];
}
const specialMap = {
'(': String.fromCharCode(0x01),
')': String.fromCharCode(0x02),
'|': String.fromCharCode(0x03),
'{': String.fromCharCode(0x04),
'}': String.fromCharCode(0x05),
',': String.fromCharCode(0x06),
'[': String.fromCharCode(0x07),
']': String.fromCharCode(0x08),
'-': String.fromCharCode(0x09),
'<': String.fromCharCode(0x0a),
'>': String.fromCharCode(0x0b),
':': String.fromCharCode(0x0c),
'=': String.fromCharCode(0x0d),
'@': String.fromCharCode(0x0e)
};
const revertSpecial = swapKeyValue(specialMap);
function swapKeyValue(object) {
const result = {};
for (const key in object) {
result[object[key]] = key;
} }
return result;
}
function escape2utfSpecial(str) {
return str.replace(/\\(.)/g, (a, chr) => specialMap[chr] ? specialMap[chr] : chr);
}
function utfSpecial2unEscaped(str) {
return str.split('').map(chr => revertSpecial[chr] ? revertSpecial[chr] : chr).join('');
}
function bracketsHandler(theString) {
// Handle []
const lower = 'abcdefghijklmnopqrstuvwxyz';
const upper = lower.toUpperCase();
const number = '0123456789';
theString = theString.replace(/(\[[^\]]*)([a-z]-[a-z])([^\]]*])/g, (osef, before, chrs, after) => before + lower.slice(lower.indexOf(chrs.split('-')[0]), lower.indexOf(chrs.split('-')[1]) + 1) + after);
theString = theString.replace(/(\[[^\]]*)([A-Z]-[A-Z])([^\]]*])/g, (osef, before, chrs, after) => before + upper.slice(upper.indexOf(chrs.split('-')[0]), upper.indexOf(chrs.split('-')[1]) + 1) + after);
theString = theString.replace(/(\[[^\]]*)([0-9]-[0-9])([^\]]*])/g, (osef, before, chrs, after) => before + number.slice(number.indexOf(chrs.split('-')[0]), number.indexOf(chrs.split('-')[1]) + 1) + after); // eslint-disable-line unicorn/better-regex
theString = theString.replace(/\[([^\]]+)]/g, (osef, chrs) => `(${chrs.split('').join('|')})`);
return theString;
}
function qtyHandlerReplaceCallback(all, chr, qty) {
const mm = qty.split(',').map(n => n.trim() * 1); // eslint-disable-line no-implicit-coercion
const min = mm[0];
const max = (mm.length === 2) ? mm[1] : min;
let result = new Array(min + 1).join(chr);// eslint-disable-line unicorn/no-new-array
for (let i = min; i < max; i++) result += `(|${chr})`;
return result;
}
function qtyHandler(theString) {
// Handle {qty} and {min,max}
theString = theString.replace(/([^)]){([^}]+)}/g, qtyHandlerReplaceCallback);
theString = theString.replace(/^(.*)\(([^)]*)\){([^}]+)}(.*)$/, (all, before, choices, qty, after) => before + qtyHandlerReplaceCallback('', `(${choices})`, qty) + after); // eslint-disable-line max-params
while(1){
const qtyFound = rightParser(theString,'{','}');
if(!qtyFound.before || !qtyFound.inside) break;
const qtyPerimeter = rightParser(qtyFound.before,'(',')');
theString = qtyPerimeter.before + qtyHandlerReplaceCallback('',`(${qtyPerimeter.inside})`,qtyFound.inside)+qtyFound.after;
}
return theString;
}
function rightParser(fullString,openLeftChr,closeRightChr) {
const chrList = fullString.split('');
const res = {
before: '',
inside: '',
after: ''
};
function insideParser(){
while (chrList.length) {
const chr = chrList.pop();
res.inside = chr+res.inside;
if(chr===openLeftChr) return;
if(chr===closeRightChr) insideParser();
}
}
while (chrList.length) {
const chr = chrList.pop();
if(chr!==closeRightChr) res.after = chr+res.after;
if(chr===closeRightChr) {
insideParser();
res.inside = res.inside.substring(1);
res.before = chrList.join('');
return res;
}
}
return res;
}
function parseAllLines(lines) { function parseAllLines(lines) {
return lines; return lines.map(line=>parseLine(line,lines));
} }
function parseLine(theLine, allLines,labelBreadCrumb='>') {
function parseLine(theLine, allLines) { if(theLine.includes('::')) return '§void§';
theLine = refHandler(theLine, allLines, labelBreadCrumb)
return theLine; return theLine;
} }
function refHandler(theString, allStrings, labelBreadCrumb) {
// Handle <ref>
theString = theString.replace(/<([^>]+)>/g, (osef, ref) => computedLabel(ref, allStrings, labelBreadCrumb));
return theString;
}
function syncRefHandler(theString, allStrings, labelBreadCrumb='>') {
// Handle =ref>
const syncRef = {};
theString.replace(/=([^>]+)>/g,(osef,ref)=> syncRef[ref] = computedLabel(ref, allStrings, labelBreadCrumb));
Object.keys(syncRef).forEach(ref=>{
const altList = [];
const altTree = tree.buildTreeStruct(syncRef[ref]);
const treeSize = tree.altCount(altTree);
for(let altIndex = 0; altIndex<treeSize; altIndex++){
const currentAlt = tree.getAlternative(altIndex,altTree);
altList.push(theString.replace(new RegExp(`=${ref}>`,'g'),currentAlt));
}
theString = flattenIt(`(${altList.join('|')})`);
});
return theString;
}
function computedLabel(label, allStrings, labelBreadCrumb) {
return cache(`label::${label}`, _computedLabel, label, allStrings,labelBreadCrumb);
}
function _computedLabel(label, allStrings, labelBreadCrumb) {
if(labelBreadCrumb.includes(label)) return '§infiniteRecursion§';
labelBreadCrumb+=label+'>';
const matchingLabel = allStrings.filter(str => str.trim().indexOf(`${label}::`) === 0);
return flattenIt(`(${matchingLabel.map(str => parseLine(labelExpressions(str), allStrings,labelBreadCrumb)).join('|')})`);
}
function labelExpressions(str) {
return str.slice(str.indexOf('::') + 2);
}
function combineUnspecified2IdSecPwd(lines) { function combineUnspecified2IdSecPwd(lines) {
const idSecPwdLines = []; const idSecPwdLines = [];
const unspecifiedLines = []; const unspecifiedLines = [];
lines.forEach(line => line.includes('@@@@') ? idSecPwdLines.push(line) : unspecifiedLines.push(line)); lines.forEach(line => line.includes('@@') ? idSecPwdLines.push(line) : unspecifiedLines.push(line));
if (unspecifiedLines.length) idSecPwdLines.push(`${mergeLines(unspecifiedLines)}@@@@${mergeLines(unspecifiedLines)}`) if (unspecifiedLines.length) idSecPwdLines.push(`${mergeLines(unspecifiedLines)}@@${mergeLines(unspecifiedLines)}`)
return mergeLines(idSecPwdLines); return mergeLines(idSecPwdLines);
} }
function mergeLines(lines) { function mergeLines(lines) {
if (!lines.length) return ''; if (!lines.length) return '';
lines = lines.filter(l => l.length); lines = lines.filter(l => l.length);
......
src/dictionary-parser.test.mjs
+ 103
− 8
View file @ 7f94cc25
import test from 'ava'; import test from 'ava';
import * as app from './dictionary-parser.mjs'; import * as app from './dictionary-parser.mjs';
test('no @@@@ idSec password separator ? implicit add it', t => { test('parse remove ref:: lines', t => {
t.is(app.parse('a'), 'a@@@@a'); t.is(app.parse('ref::truc',false), '');
}); });
test('@@@@ present, do not add it', t => { test('parse handle <ref>', t => {
t.is(app.parse('a@@@@b'), 'a@@@@b'); t.is(app.parse(`ref::truc\nref::bidule\n<ref> <ref>`,false), '(truc|bidule) (truc|bidule)');
t.is(app.parse(`ref::(truc|bidule)\n<ref> <ref>`,false), '(truc|bidule) (truc|bidule)');
}); });
test('no @@@@ on each line ? combine no @@@@ lines', t => { test('parse handle <ref> complex inclusion', t => {
t.is(app.parse(`ref::a<ref2>\n<ref>\nref2::(b|c)`,false), 'a(b|c)');
});
test('parse detect infinite recursion with <ref> and throw', t => {
t.throws(()=>app.parse(`ref::a<ref>\n<ref>`,false));
});
test('parse detect infinite recursion with nested <ref> and throw', t => {
t.throws(()=>app.parse(`ref::a<ref2>\n<ref>\nref2::b<ref>`,false));
});
test('parse handle [ -_]', t => {
t.is(app.parse('[ -_]',false), '( |-|_)');
});
test('parse handle [a-f]', t => {
t.is(app.parse('[a-f]',false), '(a|b|c|d|e|f)');
t.is(app.parse('[7-9]',false), '(7|8|9)');
t.is(app.parse('[C-F]',false), '(C|D|E|F)');
});
test('parse handle [a-c-]', t => {
t.is(app.parse('[a-c-]',false), '(a|b|c|-)');
});
test('parse handle {qty}', t => {
t.is(app.parse('a{5}',false), 'aaaaa');
});
test('parse handle {min,max}', t => {
t.is(app.parse('b{3,5}',false), 'bbb(|b)(|b)');
});
test('parse handle (string){qty}', t => {
t.is(app.parse(`c'est (toto|tata){0,2}`,false), `c'est (|toto|tata)(|toto|tata)`);
});
test('parse handle nested (s|t(ri|ng)){qty}', t => {
t.is(app.parse(`(s|t(ri|ng)){1,2}`,false), `(s|t(ri|ng))(|s|t(ri|ng))`);
});
test('parse handle plop:\\:', t => {
t.is(app.parse('plop:\\:ici',false), 'plop::ici');
t.is(app.parse('plop\\::ici',false), 'plop::ici');
t.is(app.parse('plop::ici',false), '');
});
test('parse handle [\\]*]', t => {
t.is(app.parse('[\\]*]',false), '(]|*)');
});
test('parse handle escaping common chr \\a', t => {
t.is(app.parse('\\a',false), 'a');
});
test('parse handle =ref>', t => {
t.is(app.parse(`ref::truc\nref::bidule\n=ref> =ref>`,false), '(truc truc|bidule bidule)');
t.is(app.parse(`ref::(truc|bidule)\n=ref> =ref>`,false), '(truc truc|bidule bidule)');
});
test('parse handle =ref> without generating duplication', t => {
t.is(app.parse(`ref::(truc|bidule)\n(=ref> =ref>|machin)`,false), '(truc truc|machin|bidule bidule)');
});
test('parse handle multiple =ref>', t => {
t.is(
app.parse(`=ref> =ref2> =ref> =ref2>\nref::(truc|bidule)\nref2::(machin|chose)`,false),
'(truc machin truc machin|bidule machin bidule machin|truc chose truc chose|bidule chose bidule chose)');
});
test('parse handle multi-level =ref>', t => {
t.is(app.parse(`=ref> =ref> =ref2>\nref::(truc|=ref2>)\nref2::(machin|chose)`,false),
'(truc truc machin|machin machin machin|truc truc chose|chose chose chose)');
});
test('parse throw if unconverted =ref>', t => t.throws(()=>app.parse(`=ref>\nref::=ref>`,false)));
/*TODO: test('parse handle multi-level =ref> in all case', t => {
t.is(app.parse(`=ref2> =ref> =ref>\nref::(truc|=ref2>)\nref2::(machin|chose)`,false),
'(machin truc truc|chose truc truc|machin machin machin|chose chose chose)');
});*/
test('no @@ idSec password separator ? implicit add it', t => {
t.is(app.parse('a'), 'a@@a');
});
test('@@ present, do not add it', t => {
t.is(app.parse('a@@b'), 'a@@b');
});
test('no @@ on each line ? combine no @@ lines', t => {
t.is(app.parse(` t.is(app.parse(`
a@@@@(b|c) a@@(b|c)
(d|e) (d|e)
(f|g) (f|g)
h@@@@(i|j) h@@(i|j)
`), '(a@@@@(b|c)|h@@@@(i|j)|((d|e)|(f|g))@@@@((d|e)|(f|g)))'); `), '(a@@(b|c)|h@@(i|j)|(d|e|f|g)@@(d|e|f|g))');
}); });
test('add @@ on part without it', t => {
t.is(app.parse(`(a@@b|c)`), '(a@@b|c@@c)');
});
test('add @@ on part without it complex case', t => {
t.is(app.parse(`(a(b|(c|d)@@(e|f)|g@@h|i)|(j|(k@@(l|m)n|o)))p`),
'((a((c|d)@@(e|f)|g@@h)|k@@(l|m)n)p|(a(b|i)|j|o)p@@(a(b|i)|j|o)p)');
});
test('throw if multiple @@ in the same sequence', t => t.throws(() => app.parse(`a@@(b|c)@@d`)));
test('add no accents variant', t => t.is(app.parse('Ǧ1ǦdiT ici',false,1), '(Ǧ1ǦdiT ici|G1GdiT ici)'));
test('add optional accents variants', t => t.is(app.parse('Ǧ1ǦdiT ici',false,2), '(Ǧ|G)1(Ǧ|G)diT ici'));
test('add lowercase variant', t => t.is(app.parse('Ǧ1ǦdiT ici',false,0,1), '(Ǧ1ǦdiT ici|ǧ1ǧdit ici)'));
test('add optional lowercase variant', t => t.is(app.parse('Ǧ1ǦdiT ici',false,0,2), '(Ǧ|ǧ)1(Ǧ|ǧ)di(T|t) ici'));
test('add uppercase variants', t => t.is(app.parse('Ǧ1ǦdiT ici',false,0,3), '(Ǧ1ǦdiT ici|Ǧ1ǦdiT Ici|Ǧ1ǧdit Ici|Ǧ1ǦDIT ICI)'));
//test.skip('add optional capitalized variants', t => t.is(app.parse('Ǧ1ǦdiT ici',false,0,4), '(Ǧ|ǧ)1(Ǧ|ǧ)di(T|t) (I|i)ci'));
test('add no leetSpeak variants', t => t.is(app.parse('Ǧ1ǦdiT ici',false,0,0,1), 'Ǧ(1|i|l|I|L|t|T)ǦdiT ici'));
test('add leetSpeak variants', t => t.is(app.parse('Ǧ1ǦdiT ici',false,0,0,2), '(Ǧ1ǦdiT ici|Ǧ1Ǧd1(7|1) 1c1)'));
test('add every leetSpeak variants', t => t.is(app.parse('Ǧ1ǦdiT ici',false,0,0,3), 'Ǧ(1|i|l|I|L|t|T)Ǧd(i|1|l|I|L|t|T)(T|7|t|y|Y|1|i|l|I|L) (i|1|l|I|L|t|T)c(i|1|l|I|L|t|T)'));
//test('add all variants', t => t.true(app.parse('Ǧ1ǦdiT ici',true,2,4,3).length>500));
src/dictionary-tree.mjs
+ 44
− 1
View file @ 7f94cc25
...@@ -120,7 +120,50 @@ function trivialDedup(tree) { ...@@ -120,7 +120,50 @@ function trivialDedup(tree) {
tree.forEach(v => dedupKeys[JSON.stringify(v)] = v); // eslint-disable-line no-return-assign tree.forEach(v => dedupKeys[JSON.stringify(v)] = v); // eslint-disable-line no-return-assign
return Object.keys(dedupKeys).map(JSON.parse); return Object.keys(dedupKeys).map(JSON.parse);
} }
export function splitAround(pattern,treeStruct){
function recSplitter(treeStruct){
if (isString(treeStruct)) return treeStruct.str.includes(pattern) ? {matching:treeStruct.str} : {notMatching:treeStruct.str};
if (isStep(treeStruct)) {
let isMatch = 0;
let isAlt = false;
let serialized = '';
let altSerialized = '';
treeStruct.step.forEach(subTree=>{
const subPart = recSplitter(subTree);
if(subPart.matching) {
isMatch++;
if (subPart.notMatching) isAlt = true;
serialized+=subPart.matching;
altSerialized+=subPart.notMatching;
} else {
serialized+=subPart.notMatching;
altSerialized+=subPart.notMatching;
}
});
if(isMatch>1) throw new Error('Error: @@ can only appear once in expression (idSec at left, password à right)');
if(isMatch && isAlt) return {matching:serialized, notMatching: altSerialized};
if(isMatch) return {matching:serialized};
return {notMatching: serialized};
}
if (isAlt(treeStruct)) {
const matching = [];
const notMatching = [];
treeStruct.alt.forEach(subTree=>{
const subPart = recSplitter(subTree);
if(subPart.matching) matching.push(subPart.matching);
if(subPart.notMatching) notMatching.push(subPart.notMatching);
})
if(matching.length && notMatching.length) return {matching: `(${matching.join('|')})`,notMatching: `(${notMatching.join('|')})`}
if(matching.length) return {matching: `(${matching.join('|')})`}
if(notMatching.length) return {notMatching: `(${notMatching.join('|')})`}
}
throw new Error(`Error: how to splitAround ${pattern} with ${JSON.stringify(treeStruct)} RAW: ${treeStruct}`);
}
const res = recSplitter(treeStruct);
if(res.matching) res.matching = serialize(buildTreeStruct(res.matching));
if(res.notMatching) res.notMatching = serialize(buildTreeStruct(res.notMatching));
return res;
}
export function serialize(treeStruct) { export function serialize(treeStruct) {
if (isString(treeStruct)) return treeStruct.str; if (isString(treeStruct)) return treeStruct.str;
if (isStep(treeStruct)) return treeStruct.step.map(serialize).join(''); if (isStep(treeStruct)) return treeStruct.step.map(serialize).join('');
......
src/dictionary-tree.test.mjs
+ 6
− 0
View file @ 7f94cc25
...@@ -12,10 +12,16 @@ test('(|b) keep empty choice', t => t.is(buildTreeThenSerialize('(|b)'), '(|b)') ...@@ -12,10 +12,16 @@ test('(|b) keep empty choice', t => t.is(buildTreeThenSerialize('(|b)'), '(|b)')
test('(b|b) trivial dedup', t => t.is(buildTreeThenSerialize('(|b||b|)'), '(|b)')); test('(b|b) trivial dedup', t => t.is(buildTreeThenSerialize('(|b||b|)'), '(|b)'));
test('a(b|c) mix fix and alt', t => t.is(buildTreeThenSerialize('a(b|c)'), 'a(b|c)')); test('a(b|c) mix fix and alt', t => t.is(buildTreeThenSerialize('a(b|c)'), 'a(b|c)'));
test('a(b) flat merge when no alt', t => t.is(buildTreeThenSerialize('a(b)'), 'ab')); test('a(b) flat merge when no alt', t => t.is(buildTreeThenSerialize('a(b)'), 'ab'));
test('(a(b|c)|(d|e)) flat merge when unneeded depth', t => t.is(buildTreeThenSerialize('(a(b|c)|(d|e))'), '(a(b|c)|d|e)'));
test('build complexe tree with (|) pattern', t => t.is(buildTreeThenSerialize('(a(|b@@@@c|d|)|(e|f)|g|h@@@@i)'), '(a(|b@@@@c|d)|e|f|g|h@@@@i)')); test('build complexe tree with (|) pattern', t => t.is(buildTreeThenSerialize('(a(|b@@@@c|d|)|(e|f)|g|h@@@@i)'), '(a(|b@@@@c|d)|e|f|g|h@@@@i)'));
test('serialize incorrect tree throw', t => t.throws(() => app.serialize({plop: ['a']}))); test('serialize incorrect tree throw', t => t.throws(() => app.serialize({plop: ['a']})));
test('splitAround incorrect tree throw', t => t.throws(() => app.splitAround('@@',{plop: ['a']})));
test('splitAround return notMatching case', t => t.deepEqual(app.splitAround('@',app.buildTreeStruct('a(b|c)')),{notMatching:'a(b|c)'}));
test('splitAround return matching case', t => t.deepEqual(app.splitAround('@',app.buildTreeStruct('a@b')),{matching:'a@b'}));
test('splitAround return both matching case and not matching one', t => t.deepEqual(app.splitAround('@',app.buildTreeStruct('a@b|c')),{matching:'a@b',notMatching:'c'}));
test('mono altCount', t => t.is(app.altCount(app.buildTreeStruct('ipsum')), 1)); test('mono altCount', t => t.is(app.altCount(app.buildTreeStruct('ipsum')), 1));
test('simple altCount', t => t.is(app.altCount(app.buildTreeStruct('(lore|ipsu)m')), 2)); test('simple altCount', t => t.is(app.altCount(app.buildTreeStruct('(lore|ipsu)m')), 2));
test('multi altCount', t => t.is(app.altCount(app.buildTreeStruct('(a|b|c)(d|e|f)g(h|i|j|k)')), 36)); test('multi altCount', t => t.is(app.altCount(app.buildTreeStruct('(a|b|c)(d|e|f)g(h|i|j|k)')), 36));
......
src/multi-node-layer.test.mjs
+ 3
− 0
View file @ 7f94cc25
...@@ -12,10 +12,13 @@ test('pingTracker give time elapsed', async t => { ...@@ -12,10 +12,13 @@ test('pingTracker give time elapsed', async t => {
const secondParameter = 11; const secondParameter = 11;
const toCall = [mockedFunc,firstParameter,secondParameter]; const toCall = [mockedFunc,firstParameter,secondParameter];
const timeout = 1000; const timeout = 1000;
await sleep(1); // in some test environnement this fix latency bug.
const result = await app.timeTrack(toCall,timeout); const result = await app.timeTrack(toCall,timeout);
await sleep(1); // in some test environnement this fix latency bug.
t.is(result.result,firstParameter); t.is(result.result,firstParameter);
t.is(result.params[1],firstParameter); t.is(result.params[1],firstParameter);
t.regex(result.ms.toString(),/1[0-9]/); t.regex(result.ms.toString(),/1[0-9]/);
await sleep(1); // in some test environnement this fix latency bug.
}); });
test('pingTracker can timeout', async t => { test('pingTracker can timeout', async t => {
const moreThanTimeout = 15 const moreThanTimeout = 15
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment