diff --git a/.gitignore b/.gitignore index ee6265a..f70a553 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ node_modules/ logs.log -db/image/ +db/image/*/ diff --git a/db/hashtester.js b/db/hashtester.js new file mode 100644 index 0000000..c22803e --- /dev/null +++ b/db/hashtester.js @@ -0,0 +1,19 @@ +const md5 = require('md5'); +const fs = require('fs'); + +let filename = '5241-1.png'; + +const start = Date.now(); + +filename = filename.split('.png')[0]; + +const hash = md5(filename); +console.log(hash); + +const bucket = hash.substring(0, 4); +const file = `./image/${bucket[0]}/${bucket[1]}/${bucket[2]}/${bucket[3]}/${filename}.png`; + +console.log(fs.existsSync(file)); + +const delta = Date.now() - start; +console.log(`${delta}ms`); diff --git a/db/image/compressor.sh b/db/image/compressor.sh new file mode 100644 index 0000000..af50ded --- /dev/null +++ b/db/image/compressor.sh @@ -0,0 +1,7 @@ + +for f in *; do + if [ -d "$f" ]; then + echo $f + tar -czvf $f.tar.gz "$f"/ + fi +done diff --git a/db/image/decompressor.sh b/db/image/decompressor.sh new file mode 100644 index 0000000..a2d2808 --- /dev/null +++ b/db/image/decompressor.sh @@ -0,0 +1,4 @@ + +for d in *.tar.gz ; do + tar -xvzf "$d".tar.gz -C some_custom_folder_name +done diff --git a/db/imagesorter.js b/db/imagesorter.js new file mode 100644 index 0000000..c94dac5 --- /dev/null +++ b/db/imagesorter.js @@ -0,0 +1,47 @@ +// sorts images +// Take the file name `2336p68.png`, which is a Lego "Cockpit Space Nose", +// after a simple MD5 hash, the result is: + +// ```text +// "d2ef319ea58566b55070e06096165cb8" +// ^^^^ +// ``` + +// Using the first four characters in the hash, we can allocate images +// into buckets for storage and quick retreval. This acts very similar +// to a hash table implemented in the filesystem. + +const md5 = require('md5'); +const fs = require('fs'); + +fs.readdir('./image/', (err, files) => { + files.forEach((file) => { + file = file.split('.png')[0]; + const hash = md5(file); + const bucket = hash.substring(0, 4); + const newFile = `./image/${bucket[0]}/${bucket[1]}/${bucket[2]}/${bucket[3]}/${file}.png`; + + // if directory doesn't exist, create it + if (!fs.existsSync(`./image/`)) { + fs.mkdirSync(`./image/`); + } + if (!fs.existsSync(`./image/${bucket[0]}/`)) { + fs.mkdirSync(`./image/${bucket[0]}/`); + } + if (!fs.existsSync(`./image/${bucket[0]}/${bucket[1]}/`)) { + fs.mkdirSync(`./image/${bucket[0]}/${bucket[1]}/`); + } + if (!fs.existsSync(`./image/${bucket[0]}/${bucket[1]}/${bucket[2]}/`)) { + fs.mkdirSync(`./image/${bucket[0]}/${bucket[1]}/${bucket[2]}/`); + } + if (!fs.existsSync(`./image/${bucket[0]}/${bucket[1]}/${bucket[2]}/${bucket[3]}/`)) { + fs.mkdirSync(`./image/${bucket[0]}/${bucket[1]}/${bucket[2]}/${bucket[3]}/`); + } + + fs.rename(`./image/${file}.png`, newFile, (err) => { + if (err) { + console.log(err); + } + }); + }); +}); diff --git a/package-lock.json b/package-lock.json index 9e5e4cb..6f46261 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "express": "^4.17.2", "jest": "^27.4.5", "jsdom": "^19.0.0", + "md5": "^2.3.0", "moment": "^2.29.1", "node-fetch": "^2.6.7" } @@ -1376,6 +1377,14 @@ "node": ">=10" } }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=", + "engines": { + "node": "*" + } + }, "node_modules/ci-info": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.3.0.tgz", @@ -1529,6 +1538,14 @@ "node": ">= 8" } }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=", + "engines": { + "node": "*" + } + }, "node_modules/cssom": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", @@ -2318,6 +2335,11 @@ "node": ">= 0.10" } }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, "node_modules/is-core-module": { "version": "2.8.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.0.tgz", @@ -3651,6 +3673,16 @@ "tmpl": "1.0.5" } }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -5906,6 +5938,11 @@ "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==" }, + "charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" + }, "ci-info": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.3.0.tgz", @@ -6022,6 +6059,11 @@ "which": "^2.0.1" } }, + "crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=" + }, "cssom": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", @@ -6612,6 +6654,11 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, "is-core-module": { "version": "2.8.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.0.tgz", @@ -7609,6 +7656,16 @@ "tmpl": "1.0.5" } }, + "md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "requires": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", diff --git a/package.json b/package.json index 84e0b6f..a1f767b 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "express": "^4.17.2", "jest": "^27.4.5", "jsdom": "^19.0.0", + "md5": "^2.3.0", "moment": "^2.29.1", "node-fetch": "^2.6.7" }