From 245b8fd4d14a58204075a29d6b20507b37a82b61 Mon Sep 17 00:00:00 2001
From: Ben <36240171+benkyd@users.noreply.github.com>
Date: Thu, 3 Feb 2022 19:56:09 +0000
Subject: [PATCH] the image database

Former-commit-id: ac868cf24e85a95fd65ddea93ea2324942c2cdb3
---
 .gitignore               |  2 +-
 db/hashtester.js         | 19 ++++++++++++++
 db/image/0.tar.gz        |  3 +++
 db/image/1.tar.gz        |  3 +++
 db/image/2.tar.gz        |  3 +++
 db/image/3.tar.gz        |  3 +++
 db/image/4.tar.gz        |  3 +++
 db/image/5.tar.gz        |  3 +++
 db/image/6.tar.gz        |  3 +++
 db/image/7.tar.gz        |  3 +++
 db/image/8.tar.gz        |  3 +++
 db/image/9.tar.gz        |  3 +++
 db/image/a.tar.gz        |  3 +++
 db/image/b.tar.gz        |  3 +++
 db/image/c.tar.gz        |  3 +++
 db/image/compressor.sh   |  7 +++++
 db/image/d.tar.gz        |  3 +++
 db/image/decompressor.sh |  4 +++
 db/image/e.tar.gz        |  3 +++
 db/image/f.tar.gz        |  3 +++
 db/imagesorter.js        | 47 +++++++++++++++++++++++++++++++++
 package-lock.json        | 57 ++++++++++++++++++++++++++++++++++++++++
 package.json             |  1 +
 23 files changed, 184 insertions(+), 1 deletion(-)
 create mode 100644 db/hashtester.js
 create mode 100644 db/image/0.tar.gz
 create mode 100644 db/image/1.tar.gz
 create mode 100644 db/image/2.tar.gz
 create mode 100644 db/image/3.tar.gz
 create mode 100644 db/image/4.tar.gz
 create mode 100644 db/image/5.tar.gz
 create mode 100644 db/image/6.tar.gz
 create mode 100644 db/image/7.tar.gz
 create mode 100644 db/image/8.tar.gz
 create mode 100644 db/image/9.tar.gz
 create mode 100644 db/image/a.tar.gz
 create mode 100644 db/image/b.tar.gz
 create mode 100644 db/image/c.tar.gz
 create mode 100644 db/image/compressor.sh
 create mode 100644 db/image/d.tar.gz
 create mode 100644 db/image/decompressor.sh
 create mode 100644 db/image/e.tar.gz
 create mode 100644 db/image/f.tar.gz
 create mode 100644 db/imagesorter.js

diff --git a/.gitignore b/.gitignore
index ee6265a..f70a553 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,3 @@
 node_modules/
 logs.log
-db/image/
+db/image/*/
diff --git a/db/hashtester.js b/db/hashtester.js
new file mode 100644
index 0000000..c22803e
--- /dev/null
+++ b/db/hashtester.js
@@ -0,0 +1,19 @@
+const md5 = require('md5');
+const fs = require('fs');
+
+let filename = '5241-1.png';
+
+const start = Date.now();
+
+filename = filename.split('.png')[0];
+
+const hash = md5(filename);
+console.log(hash);
+
+const bucket = hash.substring(0, 4);
+const file = `./image/${bucket[0]}/${bucket[1]}/${bucket[2]}/${bucket[3]}/${filename}.png`;
+
+console.log(fs.existsSync(file));
+
+const delta = Date.now() - start;
+console.log(`${delta}ms`);
diff --git a/db/image/0.tar.gz b/db/image/0.tar.gz
new file mode 100644
index 0000000..95a84a5
--- /dev/null
+++ b/db/image/0.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fb20220571d719c21b348a4fb2697bb25f1d3629e317b93bb12e0d8c29df67e
+size 969172739
diff --git a/db/image/1.tar.gz b/db/image/1.tar.gz
new file mode 100644
index 0000000..af4bd43
--- /dev/null
+++ b/db/image/1.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39cce49c2b08db10690e6dbdd5c90d7bd732392645225af3ff01967912592b5b
+size 978392963
diff --git a/db/image/2.tar.gz b/db/image/2.tar.gz
new file mode 100644
index 0000000..6ee7b63
--- /dev/null
+++ b/db/image/2.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb76405cb8d491ff883ad57cca2c91bd31d575bac4c7dc2e2629275ed36c253f
+size 1019840244
diff --git a/db/image/3.tar.gz b/db/image/3.tar.gz
new file mode 100644
index 0000000..26bf996
--- /dev/null
+++ b/db/image/3.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c44991b1d3d5755c7c98428021ccf657c30337424d877ec487371785379e010
+size 998324354
diff --git a/db/image/4.tar.gz b/db/image/4.tar.gz
new file mode 100644
index 0000000..e8c21d2
--- /dev/null
+++ b/db/image/4.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2736b022f6b63358f396024799a0642e80762ccd7b75d0382fa92989eacb165
+size 971438151
diff --git a/db/image/5.tar.gz b/db/image/5.tar.gz
new file mode 100644
index 0000000..6bd21ed
--- /dev/null
+++ b/db/image/5.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44132b34229096d5a393179527af4d673684bf51ce9a69ee8115053bed670159
+size 1018434330
diff --git a/db/image/6.tar.gz b/db/image/6.tar.gz
new file mode 100644
index 0000000..75d8b1b
--- /dev/null
+++ b/db/image/6.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1cca83ab09f639000c034e93f431b9f5564391f7348b7c86be6692c7e0db919
+size 1007458122
diff --git a/db/image/7.tar.gz b/db/image/7.tar.gz
new file mode 100644
index 0000000..e873ade
--- /dev/null
+++ b/db/image/7.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8307204d755d62ff4740eccdea82cd7174beffb8bd8f6ec847798a7ae5eec081
+size 983908427
diff --git a/db/image/8.tar.gz b/db/image/8.tar.gz
new file mode 100644
index 0000000..59015e5
--- /dev/null
+++ b/db/image/8.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60e8db8df71be316e8c595fa71681c6c831ef545b038ef375a627cff8ef5a361
+size 986151083
diff --git a/db/image/9.tar.gz b/db/image/9.tar.gz
new file mode 100644
index 0000000..7ecb010
--- /dev/null
+++ b/db/image/9.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4ffbb714f2f07e17aa64323a33000efaf7748c2aab31bda545c7546c824c6b5
+size 1002302585
diff --git a/db/image/a.tar.gz b/db/image/a.tar.gz
new file mode 100644
index 0000000..7cc527b
--- /dev/null
+++ b/db/image/a.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15af4d1d35fd8ed60764ed068e3be1293619244e80eea4e672203140ba4a1301
+size 979253154
diff --git a/db/image/b.tar.gz b/db/image/b.tar.gz
new file mode 100644
index 0000000..be162b5
--- /dev/null
+++ b/db/image/b.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:868d32ba5b8a8f11f35c27d8f8d1707293674f18830922058d50d325ae9c4004
+size 1009976899
diff --git a/db/image/c.tar.gz b/db/image/c.tar.gz
new file mode 100644
index 0000000..d1110a0
--- /dev/null
+++ b/db/image/c.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90f96f74d081679b5279862299a6910a11cbc791d371e95b56c8b56a16e4efb2
+size 974228333
diff --git a/db/image/compressor.sh b/db/image/compressor.sh
new file mode 100644
index 0000000..af50ded
--- /dev/null
+++ b/db/image/compressor.sh
@@ -0,0 +1,7 @@
+
+for f in *; do
+    if [ -d "$f" ]; then
+        echo $f
+        tar -czvf $f.tar.gz "$f"/
+    fi
+done
diff --git a/db/image/d.tar.gz b/db/image/d.tar.gz
new file mode 100644
index 0000000..a3eb41e
--- /dev/null
+++ b/db/image/d.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:289c78c17f34ecfbfe7aaac3f9756806ae54f4c957018882b2e638d6f5c2b8e5
+size 1020586118
diff --git a/db/image/decompressor.sh b/db/image/decompressor.sh
new file mode 100644
index 0000000..a2d2808
--- /dev/null
+++ b/db/image/decompressor.sh
@@ -0,0 +1,4 @@
+
+for d in *.tar.gz ; do
+    tar -xvzf "$d".tar.gz -C some_custom_folder_name
+done
diff --git a/db/image/e.tar.gz b/db/image/e.tar.gz
new file mode 100644
index 0000000..2d1a1c8
--- /dev/null
+++ b/db/image/e.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1fa91e64c837584894fbdbd6b94b65d672aa4538aea6ee85748e35be30b1403
+size 1008906926
diff --git a/db/image/f.tar.gz b/db/image/f.tar.gz
new file mode 100644
index 0000000..08ea185
--- /dev/null
+++ b/db/image/f.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfba2d65311ee7b407b69419c837c45f70f0cede955a824bb5722e3f61c04aa3
+size 998828122
diff --git a/db/imagesorter.js b/db/imagesorter.js
new file mode 100644
index 0000000..c94dac5
--- /dev/null
+++ b/db/imagesorter.js
@@ -0,0 +1,47 @@
+// sorts images
+// Take the file name `2336p68.png`, which is a Lego "Cockpit Space Nose",
+// after a simple MD5 hash, the result is:
+
+// ```text
+// "d2ef319ea58566b55070e06096165cb8"
+//  ^^^^
+// ```
+
+// Using the first four characters in the hash, we can allocate images
+// into buckets for storage and quick retreval. This acts very similar
+// to a hash table implemented in the filesystem.
+
+const md5 = require('md5');
+const fs = require('fs');
+
+fs.readdir('./image/', (err, files) => {
+    files.forEach((file) => {
+        file = file.split('.png')[0];
+        const hash = md5(file);
+        const bucket = hash.substring(0, 4);
+        const newFile = `./image/${bucket[0]}/${bucket[1]}/${bucket[2]}/${bucket[3]}/${file}.png`;
+        
+        // if directory doesn't exist, create it
+        if (!fs.existsSync(`./image/`)) {
+            fs.mkdirSync(`./image/`);
+        }
+        if (!fs.existsSync(`./image/${bucket[0]}/`)) {
+            fs.mkdirSync(`./image/${bucket[0]}/`);
+        }
+        if (!fs.existsSync(`./image/${bucket[0]}/${bucket[1]}/`)) {
+            fs.mkdirSync(`./image/${bucket[0]}/${bucket[1]}/`);
+        }
+        if (!fs.existsSync(`./image/${bucket[0]}/${bucket[1]}/${bucket[2]}/`)) {
+            fs.mkdirSync(`./image/${bucket[0]}/${bucket[1]}/${bucket[2]}/`);
+        }
+        if (!fs.existsSync(`./image/${bucket[0]}/${bucket[1]}/${bucket[2]}/${bucket[3]}/`)) {
+            fs.mkdirSync(`./image/${bucket[0]}/${bucket[1]}/${bucket[2]}/${bucket[3]}/`);
+        }
+
+        fs.rename(`./image/${file}.png`, newFile, (err) => {
+            if (err) {
+                console.log(err);
+            }
+        });
+    });
+});
diff --git a/package-lock.json b/package-lock.json
index 9e5e4cb..6f46261 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -15,6 +15,7 @@
         "express": "^4.17.2",
         "jest": "^27.4.5",
         "jsdom": "^19.0.0",
+        "md5": "^2.3.0",
         "moment": "^2.29.1",
         "node-fetch": "^2.6.7"
       }
@@ -1376,6 +1377,14 @@
         "node": ">=10"
       }
     },
+    "node_modules/charenc": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
+      "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=",
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/ci-info": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.3.0.tgz",
@@ -1529,6 +1538,14 @@
         "node": ">= 8"
       }
     },
+    "node_modules/crypt": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
+      "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=",
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/cssom": {
       "version": "0.5.0",
       "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
@@ -2318,6 +2335,11 @@
         "node": ">= 0.10"
       }
     },
+    "node_modules/is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+    },
     "node_modules/is-core-module": {
       "version": "2.8.0",
       "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.0.tgz",
@@ -3651,6 +3673,16 @@
         "tmpl": "1.0.5"
       }
     },
+    "node_modules/md5": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
+      "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
+      "dependencies": {
+        "charenc": "0.0.2",
+        "crypt": "0.0.2",
+        "is-buffer": "~1.1.6"
+      }
+    },
     "node_modules/media-typer": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
@@ -5906,6 +5938,11 @@
       "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz",
       "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw=="
     },
+    "charenc": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
+      "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc="
+    },
     "ci-info": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.3.0.tgz",
@@ -6022,6 +6059,11 @@
         "which": "^2.0.1"
       }
     },
+    "crypt": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
+      "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs="
+    },
     "cssom": {
       "version": "0.5.0",
       "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
@@ -6612,6 +6654,11 @@
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
       "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="
     },
+    "is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+    },
     "is-core-module": {
       "version": "2.8.0",
       "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.0.tgz",
@@ -7609,6 +7656,16 @@
         "tmpl": "1.0.5"
       }
     },
+    "md5": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
+      "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
+      "requires": {
+        "charenc": "0.0.2",
+        "crypt": "0.0.2",
+        "is-buffer": "~1.1.6"
+      }
+    },
     "media-typer": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
diff --git a/package.json b/package.json
index 84e0b6f..a1f767b 100644
--- a/package.json
+++ b/package.json
@@ -18,6 +18,7 @@
     "express": "^4.17.2",
     "jest": "^27.4.5",
     "jsdom": "^19.0.0",
+    "md5": "^2.3.0",
     "moment": "^2.29.1",
     "node-fetch": "^2.6.7"
   }