WiP : release maybe, dictionary cache + small tweak/fix

.gitlab-ci.yml

@@ -2,6 +2,7 @@ stages:
   - build
   - test
   - publish
+  - release
 
 build:
   stage: build
@@ -59,10 +60,14 @@ pages:
   stage: test
   image: node:latest
   coverage: '/Statements[^0-9]+(\d+\.\d+)%/'
+  variables:
+    VERSION: "$CI_COMMIT_TAG || 'latest'"
   script:
     - npm run test:production
     - mkdir -p public/dist
     - cp -rf generated/npm/* public/dist/
+    - 'cp generated/npm/browser/all.mjs public/dist/browser/$VERSION.mjs'
+    - 'cp generated/npm/nodejs/all.mjs public/dist/nodejs/$VERSION.mjs'
     - mv generated/coverage public/
     - mv generated/jscpd/html public/jscpd
     - mv generated/maintainability public/
@@ -85,5 +90,38 @@ npm:
     - cd ./generated/npm
     - echo '//registry.npmjs.org/:_authToken=${NPM_TOKEN}'>.npmrc
     - npm publish
+  artifacts:
+    untracked: true
+    name: '$CI_COMMIT_TAG_for_nodejs'
+    paths:
+      - generated/npm/nodejs/
+  only:
+    - tags
+
+##--------------------------------RELEASE-------------------------------------------------------
+release:
+  stage: release
+  image: registry.gitlab.com/gitlab-org/release-cli:latest
+  script:
+    - cd ./generated/npm
+    - echo '//registry.npmjs.org/:_authToken=${NPM_TOKEN}'>.npmrc
+    - npm publish
+  release:
+    tag_name: '$CI_COMMIT_TAG'
+    name: '$CI_COMMIT_TAG'
+    description: '$CI_COMMIT_MESSAGE'
+    assets:
+      links:
+        - name: 'g1lib_for_browser'
+          url: 'https://libs.duniter.io/g1lib.js/public/dist/browser/$CI_COMMIT_TAG.mjs'
+          link_type: 'package'
+        - name: 'g1lib_for_nodejs'
+          url: 'https://libs.duniter.io/g1lib.js/public/dist/nodejs/$CI_COMMIT_TAG.mjs'
+          link_type: 'package'
+  artifacts:
+    untracked: true
+    name: '$CI_COMMIT_TAG_for_browser'
+    paths:
+      - generated/npm/browser/
   only:
     - tags

CHANGELOG.fr.md

@@ -7,12 +7,24 @@ Le format est basé sur [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 et ce projet adhère au [versionnage sémantique](https://semver.org/spec/v2.0.0.html).
 
 ## Evolutions probable / Roadmap :
-- GraphQL stuff
-- @@@@ comme séparateur entre identifiant secret et mdp pour la génération de combinaison à tester (usage principal Gsper)
-
+- HD wallet (dérivation)
+- sharding
+- udid / civil_id_hash
+- GVA && indexer v2s GraphQL stuff (gestion des paiements, détection des paiements...)
 
 ## [Non-publié/Non-Stabilisé] (par [1000i100])
 
+## [Version 3.5.0] - 2022-11-27 (par [1000i100])
+### Ajouté
+- dictionary intègre un mécanisme de cache pour détecter les doublons. Il est désactivable pour éviter les crashs par saturation de la mémoire.
+- dictionary-parser gère toutes les syntaxes d'expression régulières qu'utilisaient gsper v2 + des situations plus complexes
+- dictionary-parser permet de distinguer identifiant secret et mot de passe via le séparateur `@@`
+- dictionary-parser permet plusieurs types de déclinaisons : sans accents, accents optionnels, sans majuscule, majuscule optionnelle, tout en majuscule, et des déclinaison type leetSpeak.
+- dictionary-tree permet de savoir combien de combinaisons sont possibles et d'itérer dessus sans avoir besoin de les pré-générer.
+### Corrections
+- Vérification à chaque build (et donc dans la CI) que les packets destinés à tourner dans le navigateur n'ont aucunes dépendances.
+- Suppression des dépendances résiduelles.
+
 ## [Version 3.4.2] - 2022-11-20 (par [1000i100])
 ### Corrections
 - checkKey ne complète plus automatiquement les clef trop courtes et envoi donc l'erreur attendue pour les clefs trops courtes.
@@ -109,8 +121,9 @@ et ce projet adhère au [versionnage sémantique](https://semver.org/spec/v2.0.0
 - intégration des librairies de crypto nécessaires
 - calcul de la clef publique correspondant à chaque combinaison de secrets saisie, et comparaison à la clef publique de référence.
 
-[Non-publié/Non-Stabilisé]: https://git.duniter.org/libs/g1lib.js/-/compare/v3.4.2...main
+[Non-publié/Non-Stabilisé]: https://git.duniter.org/libs/g1lib.js/-/compare/v3.5.0...main
 
+[Version 3.5.0]: https://git.duniter.org/libs/g1lib.js/-/compare/v3.4.2...v3.5.0
 [Version 3.4.2]: https://git.duniter.org/libs/g1lib.js/-/compare/v3.4.1...v3.4.2
 [Version 3.4.1]: https://git.duniter.org/libs/g1lib.js/-/compare/v3.4.0...v3.4.1
 [Version 3.4.0]: https://git.duniter.org/libs/g1lib.js/-/compare/v3.3.3...v3.4.0

npm/package.json

 {
   "name": "g1lib",
-  "version": "3.4.2",
+  "version": "3.5.0",
   "description": "An ubiquitous static javascript toolbox lib for Ǧ1 / Duniter ecosystem with reliability in mind.",
   "main": "nodejs/all.mjs",
 	"browser": "browser/all.mjs",

src/dictionary-parser.mjs

@@ -128,7 +128,7 @@ function toLeet(str){
 		'R': '2',
 		'Z': '2',
 		'E': '3',
-		'A': '(4|@)',
+		'A': '4', // '(4|@)' conflit avec la syntaxe @@
 		'S': '(5|$)',
 		'G': '(6|9)',
 		'T': '(7|1)',
@@ -138,7 +138,7 @@ function toLeet(str){
 }
 function fromLeet(str){
 	return replaceList(str,{
-		'@': '(a|A)',
+		//'@': '(a|A)', conflit avec la syntaxe @@
 		'$': '(s|S)',
 		'0': '(o|O)',
 		'1': '(i|l|I|L|t|T)',

src/dictionary-tree.mjs

-export function build(dictionaryString) {
-}
-
 export function buildTreeStruct(monoLineString) {
 	const stringAsArray = monoLineString.split('');
 	const rawTree = leftParser(stringAsArray);

src/dictionary.mjs

+import {buildTreeStruct, getAlternative, serialize} from "./dictionary-tree.mjs";
+import {parse} from "./dictionary-parser.mjs";
+
 export class Dictionary {
-	constructor(dictionaryString) {
-		this.properties = initProperties(dictionaryString);
-		incorporate(this, this.properties);
+	constructor(dictionaryString, options= {}) {
+		this.originalConfig = options;
+		this.originalConfig.dictionaryString = dictionaryString;
+		this.config = JSON.parse(JSON.stringify(this.originalConfig));
+		if(typeof this.config.cache === 'undefined') this.config.cache = true;
+		if(typeof this.config.idSecPwd === 'undefined') this.config.idSecPwd = true;
+		this.tree = buildTreeStruct(parse(dictionaryString,this.config.idSecPwd));
+		this.estimateDuration = ()=>this.tree.altCount/(this.config.speed || 1)
+		this.estimateRemaining = ()=>(this.tree.altCount-this.tried)/(this.config.speed || 1)
+		if(this.config.speed) adjustVariant(this);
+		this.length = this.tree.altCount;
+		this.tried = 0;
+		this.cache = [];
+		this.duplicatedCount = ()=> this.tried - Object.keys(this.cache).length;
+		this.dryGet = index => getAlternative(index,this.tree);
+		this.get = index => {
+			if(typeof this.startTime === "undefined") this.startTime = Date.now();
+			this.tried++;
+			const alt = getAlternative(index,this.tree);
+			if(this.config.cache) {
+				if(typeof this.cache[alt] === "undefined") this.cache[alt] = [];
+				this.cache[alt].push(index);
+				if(this.cache[alt].length>1) return `§duplicate§${alt}`;
 			}
+			return alt;
 		}
-
-export function initProperties(dictionaryString) {
-	if (!dictionaryString) return {};
-	const properties = {
-		length: 81,
-		extraLength: 90
-	};
-	return properties;
+		this.timeSpent = ()=>(Date.now()-this.startTime)/1000;
+		this.duplicatedFound = ()=>{
+			const duplicated = [];
+			for(let key of Object.keys(this.cache)) if(this.cache[key].length>1) duplicated.push({alt:key,index:this.cache[key]});
+			const sortedDuplicate = duplicated.sort((a,b)=>b.index.length-a.index.length);
+			return sortedDuplicate;
 		}
-
-export default Dictionary;
-
-function incorporate(target, toAdd) {
-	for (const key in toAdd) {
-		target[key] = toAdd[key];
+		this.dryRunDedup = ()=>dryRunDedup(this);
+	}
+}
+function adjustVariant(self){
+	const durationGoal = 3_600;
+	if (self.estimateDuration() >= durationGoal) return;
+	function rebuildTree(self){
+		self.tree = buildTreeStruct(parse(self.originalConfig.dictionaryString,self.config.idSecPwd,self.config.accent,self.config.lowerCase,self.config.leetSpeak));
+	}
+	let lastLess1hConfig;
+	while (self.estimateDuration() < durationGoal){
+		lastLess1hConfig = JSON.parse(JSON.stringify(self.config));
+		if(self.config.accent !== 2 && (typeof self.originalConfig.accent === 'undefined' || self.originalConfig.accent === "auto")) {
+			if(!self.config.accent) self.config.accent = 1;
+			else self.config.accent++;
+			rebuildTree(self);
+			continue;
+		}
+		if(self.config.lowerCase !== 4 && (typeof self.originalConfig.lowerCase === 'undefined' || self.originalConfig.lowerCase === "auto")) {
+			if(!self.config.lowerCase) self.config.lowerCase = 1;
+			else self.config.lowerCase++;
+			rebuildTree(self);
+			continue;
+		}
+		if(self.config.leetSpeak !== 3 && (typeof self.originalConfig.leetSpeak === 'undefined' || self.originalConfig.leetSpeak === "auto")) {
+			if(!self.config.leetSpeak) self.config.leetSpeak = 1;
+			else self.config.leetSpeak++;
+			rebuildTree(self);
+			continue;
+		}
+		console.log('strange, not an hour yet with all variants ?');
+		break;
+	}
+	self.config = lastLess1hConfig;
+	rebuildTree(self);
+	dryRunDedup(self);
 }
+function dryRunDedup(self){
+	const dry = new Dictionary(serialize(self.tree),{cache:true,idSecPwd:false});
+	for(let i = 0; i < dry.length;i++) dry.get(i);
+	const duplicate = dry.duplicatedFound();
+	self.lengthBeforeDedup = dry.length;
+	//TODO: Factorize dry.cache keys
+	const dedupAsString = `(${Object.keys(dry.cache).join('|')})`;
+	self.tree = buildTreeStruct(dedupAsString,false);
+	self.config.cache = false;
+	self.length = self.tree.altCount;
+	return duplicate;
 }

src/dictionary.test.mjs

@@ -2,12 +2,75 @@ import test from 'ava';
 import * as app from './dictionary.mjs';
 
 test('get dictionary length', t => {
+	const dictionaryString = '(a|b|c)d(e|f|g)';
+	const dico = new app.Dictionary(dictionaryString,{idSecPwd:false});
+	t.is(dico.length, 9);
+});
+test('get dictionary iteration', t => {
+	const dictionaryString = '(a|b|c)d(e|f|g)';
+	const dico = new app.Dictionary(dictionaryString);
+	t.is(dico.get(5), "ade@@bdg");
+});
+test('get past iteration count in this dictionary', t => {
 	const dictionaryString = '(a|b|c)d(e|f|g)';
 	const dico = new app.Dictionary(dictionaryString);
-	t.is(dico.length, 81);
+	t.is(dico.tried, 0);
+	dico.get(1);
+	dico.get(2);
+	t.is(dico.tried, 2);
 });
-test('get dictionary extraLength', t => {
+test('get duplicated found count (from dictionary)', t => {
 	const dictionaryString = '(a|b|c)d(e|f|g)';
 	const dico = new app.Dictionary(dictionaryString);
-	t.true(dico.extraLength >= 81);
+	dico.get(0);
+	dico.get(1);
+	t.is(dico.duplicatedCount(), 0);
+});
+test('get duplicated found in past iteration (from dictionary)', t => {
+	const dictionaryString = '((a|b)cd|(a|b)c(d|e)|bc(d|e))';
+	const dico = new app.Dictionary(dictionaryString, {idSecPwd:false});
+	for(let i = 0; i < dico.length;i++) dico.get(i);
+	t.deepEqual(dico.duplicatedFound(), [ {alt:"bcd",index: [ 1, 4, 6 ]}, {alt:"acd",index: [ 0, 2 ]}, {alt:"bce",index: [ 5, 7 ]} ]);
+});
+test('dictionary can dryRun to find all duplicate', t => {
+	const dictionaryString = '((a|b)cd|(a|b)c(d|e)|bc(d|e))';
+	const dico = new app.Dictionary(dictionaryString, {idSecPwd:false});
+	const duplicate = dico.dryRunDedup();
+	t.deepEqual(duplicate, [ {alt:"bcd",index: [ 1, 4, 6 ]}, {alt:"acd",index: [ 0, 2 ]}, {alt:"bce",index: [ 5, 7 ]} ]);
+	t.is(dico.lengthBeforeDedup,8);
+	t.is(dico.length,4);
+});
+
+test('dictionary can run with cache disabled', t => {
+	const dictionaryString = '(a|b|c)d(e|f|g)';
+	const dico = new app.Dictionary(dictionaryString,{cache:false});
+	dico.get(0);
+	t.is(dico.duplicatedCount(), dico.tried);
+});
+test('dictionary can run with cache on', t => {
+	const dictionaryString = 'a';
+	const dico = new app.Dictionary(dictionaryString,{cache:true});
+	dico.get(0);
+	dico.get(0);
+	t.is(dico.duplicatedCount(), 1);
+});
+test('duplicate match §duplicate§ pattern', t => {
+	const dictionaryString = 'a';
+	const dico = new app.Dictionary(dictionaryString);
+	const first = dico.get(0);
+	const second = dico.get(0);
+	t.is(first, 'a@@a');
+	t.is(second, '§duplicate§a@@a');
+});
+test.skip('dictionary called with speed option try to activate variante accent, caps and leetSpeak option to reach 1h of compute estimated time', t => {
+	const dictionaryString = 'Ǧ1Ǧ1';
+	const dico = new app.Dictionary(dictionaryString, {speed:30, leetSpeak:1});
+	t.is(dico.lengthBeforeDedup,2);
+	t.is(dico.length,2);
+});
+test.skip('dico should not crash', t=>{
+	const v8CrashStringButFirefoxWork = '(((Ǧ|ǧ)|(G|g))(1|i|l|I)((Ǧ|ǧ)|(G|g))(1|i|l|I)@@((Ǧ|ǧ)|(G|g))(1|i|l|I)((Ǧ|ǧ)|(G|g))(1|i|l|I)|(((Ǧ|ǧ)|(G|g))(1|i|l|I)((Ǧ|ǧ)|(G|g))(1|i|l|I)@@((Ǧ|ǧ)|(G|g))(1|i|l|I)((Ǧ|ǧ)|(G|g))(1|i|l|I)|(ǧ|g)(1|i|l|I)(ǧ|g)(1|i|l|I)@@(ǧ|g)(1|i|l|I)(ǧ|g)(1|i|l|I)))';
+	const dico = new app.Dictionary(v8CrashStringButFirefoxWork,{speed:30})
+	//t.is(dico.lengthBeforeDedup,2);
+	t.is(dico.length,2);
 });