benkyd/legolog
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

databases

Browse Source 
Former-commit-id: 5bc8a1b761fe2241c32e51b30b6c0e0dfdeb3461
This commit is contained in:
Ben
2022-02-03 18:55:09 +00:00
parent 68933edf94
commit 925af9cc9d
12 changed files with 1498 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitattributes vendored Normal file
View File

@@ -0,0 +1 @@
*.tar.gz filter=lfs diff=lfs merge=lfs -text

159
db/LegoLog Schema Proj.dbs
View File

@@ -1,8 +1,8 @@
<?xml version="1.0" encoding="UTF-8" ?>
<project name="AppEng 2021/22" id="Project-1f6c" database="Sqlite" >
<schema name="AppEng 2021/22" >
<table name="Brick" prior="LegoBrick" >
<column name="id" type="varchar" length="20" jt="12" mandatory="y" />
<project name="schema.sql" id="Project-323" database="Sqlite" >
<schema name="public" >
<table name="Brick" spec="" >
<column name="id" type="varchar" length="50" jt="12" mandatory="y" />
<column name="name" type="text" length="100" jt="-1" mandatory="y" />
<column name="colour" type="integer" jt="4" />
<column name="catagory" type="integer" jt="4" />
@@ -10,70 +10,143 @@
<column name="dimensions_x" type="integer" jt="4" />
<column name="dimensions_y" type="integer" jt="4" />
<column name="dimensions_z" type="integer" jt="4" />
<index name="pk_LegoBrick" unique="PRIMARY_KEY" >
<column name="date_from" type="date" jt="91" />
<column name="date_to" type="date" jt="91" />
<index name="pk_Brick" unique="PRIMARY_KEY" >
<column name="id" />
</index>
<fk name="fk_colour" to_schema="AppEng 2021/22" to_table="BrickColour" >
<fk name="FK_Brick BrickColour" to_schema="public" to_table="BrickColour" options="" >
<fk_column name="colour" pk="id" />
</fk>
<fk name="fk_catagory" to_schema="AppEng 2021/22" to_table="Catagory" >
<fk name="FK_Brick Catagory" to_schema="public" to_table="Catagory" options="" >
<fk_column name="catagory" pk="id" />
</fk>
</table>
<table name="BrickColour" prior="Brc" >
<table name="BrickColour" spec="" >
<column name="id" type="integer" jt="4" mandatory="y" />
<column name="name" type="varchar" length="100" jt="12" />
<column name="hexrgb" type="varchar" length="6" jt="12" mandatory="y" />
<column name="type" prior="is_transparent" type="integer" jt="4" />
<column name="date_from" type="date" jt="91" />
<column name="date_to" type="date" jt="91" />
<column name="col_type" type="integer" jt="4" />
<index name="pk_BrickColour" unique="PRIMARY_KEY" >
<column name="id" />
</index>
<fk name="fk_ColourType" to_schema="AppEng 2021/22" to_table="ColourType" >
<fk_column name="type" pk="id" />
<fk name="FK_BrickColour ColourType" to_schema="public" to_table="ColourType" options="" >
<fk_column name="col_type" pk="id" />
</fk>
</table>
<table name="Catagory" prior="Tbl_0" >
<column name="id" type="integer" jt="4" mandatory="y" />
<column name="name" type="varchar" length="100" jt="12" />
<index name="pk_Catagory" unique="PRIMARY_KEY" >
<column name="id" />
</index>
<table name="BrickInventory" >
<column name="id" type="varchar" length="50" jt="12" />
</table>
<table name="ColourType" prior="Tbl" >
<column name="id" type="integer" jt="4" mandatory="y" />
<column name="type" type="varchar" length="64" jt="12" />
<index name="pk_ColourType" unique="PRIMARY_KEY" >
<column name="id" />
</index>
</table>
<table name="Set" prior="Tbl" >
<column name="id" type="varchar" length="20" jt="12" mandatory="y" />
<table name="BrickSet" spec="" >
<column name="id" type="varchar" length="50" jt="12" mandatory="y" />
<column name="catagory" type="integer" jt="4" />
<column name="name" type="varchar" length="100" jt="12" />
<column name="date_released" type="date" jt="91" />
<column name="dimensions_x" type="decimal" jt="3" />
<column name="dimensions_y" type="decimal" jt="3" />
<column name="dimensions_z" type="decimal" jt="3" />
<fk name="fk_catagory" to_schema="AppEng 2021/22" to_table="Catagory" >
<fk name="FK_BrickSet Catagory" to_schema="public" to_table="Catagory" options="" >
<fk_column name="catagory" pk="id" />
</fk>
</table>
<table name="BrickSetInventory" >
<column name="id" type="varchar" length="50" jt="12" />
</table>
<table name="Catagory" spec="" >
<column name="id" type="integer" jt="4" mandatory="y" />
<column name="name" type="varchar" length="100" jt="12" />
<index name="pk_Catagory" unique="PRIMARY_KEY" >
<column name="id" />
</index>
</table>
<table name="ColourType" spec="" >
<column name="id" type="integer" jt="4" mandatory="y" />
<column name="name" type="varchar" length="64" jt="12" />
<index name="pk_ColourType" unique="PRIMARY_KEY" >
<column name="id" />
</index>
</table>
<table name="SetBrick" spec="" >
<column name="id" type="varchar" length="50" jt="12" mandatory="y" />
<column name="set_id" type="varchar" length="50" jt="12" mandatory="y" />
<column name="brick_id" type="varchar" length="50" jt="12" mandatory="y" />
<column name="amount" type="integer" jt="4" />
<index name="pk_SetBrick" unique="PRIMARY_KEY" >
<column name="id" />
</index>
<fk name="FK_SetBrick BrickSet" to_schema="public" to_table="BrickSet" options="" >
<fk_column name="set_id" pk="id" />
</fk>
<fk name="FK_SetBrick Brick" to_schema="public" to_table="Brick" options="" >
<fk_column name="brick_id" pk="id" />
</fk>
</table>
</schema>
<connector name="MyDb" database="MySql" host="localhost" port="3306" user="root" />
<layout name="Default Layout" id="Layout-de8" confirmed="y" show_relation="columns" >
<entity schema="AppEng 2021/22" name="Brick" color="C1D8EE" x="48" y="80" />
<entity schema="AppEng 2021/22" name="BrickColour" color="C1D8EE" x="320" y="80" />
<entity schema="AppEng 2021/22" name="Catagory" color="C1D8EE" x="336" y="272" />
<entity schema="AppEng 2021/22" name="ColourType" color="C1D8EE" x="560" y="80" />
<entity schema="AppEng 2021/22" name="Set" color="C1D8EE" x="64" y="384" />
<callout x="752" y="48" pointer="Round" >
<comment><![CDATA[Right-click the layout to create entities or shapes, create relations using drag and drop.
Save the design to file. Use the Convert menu option to convert the model to a physical model.]]></comment>
</callout>
<browser id="Browse-1dc0" name="type" confirm_updates="y" confirmed="y" >
<browse_table schema="AppEng 2021/22" entity="BrickColour" x="20" y="20" width="400" height="300" />
</browser>
<connector name="Sqlite" database="Sqlite" driver_class="org.sqlite.JDBC" driver_jar="sqlite-jdbc-3.36.0.3.jar" driver_desc="Standard" host="localhost" port="7210" />
<layout name="Layout" id="Layout-20e8" show_column_type="y" show_relation="columns" >
<entity schema="public" name="Brick" color="BED3F4" x="272" y="48" />
<entity schema="public" name="BrickColour" color="BED3F4" x="512" y="48" />
<entity schema="public" name="BrickInventory" color="C1D8EE" x="848" y="272" />
<entity schema="public" name="BrickSet" color="BED3F4" x="272" y="352" />
<entity schema="public" name="BrickSetInventory" color="C1D8EE" x="848" y="352" />
<entity schema="public" name="Catagory" color="BED3F4" x="464" y="256" />
<entity schema="public" name="ColourType" color="BED3F4" x="720" y="48" />
<entity schema="public" name="SetBrick" color="BED3F4" x="48" y="256" />
<script name="schema.sql" id="Editor-1de1" language="SQL" >
<string><![CDATA[CREATE TABLE Catagory (
id integer NOT NULL PRIMARY KEY ,
name varchar(100)
);
CREATE TABLE ColourType (
id integer NOT NULL PRIMARY KEY ,
name varchar(64)
);
CREATE TABLE BrickSet (
id varchar(50) NOT NULL ,
catagory integer ,
name varchar(100) ,
date_released date ,
dimensions_x decimal ,
dimensions_y decimal ,
dimensions_z decimal ,
FOREIGN KEY ( catagory ) REFERENCES Catagory( id )
);
CREATE TABLE SetBrick (
id varchar(50) NOT NULL PRIMARY KEY ,
set_id varchar(50) NOT NULL ,
brick_id varchar(50) NOT NULL ,
amount integer ,
FOREIGN KEY ( set_id ) REFERENCES BrickSet( id ) ,
FOREIGN KEY ( brick_id ) REFERENCES Brick( id ) ,
);
CREATE TABLE BrickColour (
id integer NOT NULL PRIMARY KEY ,
name varchar(100) ,
hexrgb varchar(6) NOT NULL ,
col_type integer ,
FOREIGN KEY ( col_type ) REFERENCES ColourType( id )
);
CREATE TABLE Brick (
id varchar(50) NOT NULL PRIMARY KEY ,
name text(100) NOT NULL ,
colour integer ,
catagory integer ,
weight decimal ,
dimensions_x integer ,
dimensions_y integer ,
dimensions_z integer ,
date_from date ,
date_to date ,
FOREIGN KEY ( colour ) REFERENCES BrickColour( id ) ,
FOREIGN KEY ( catagory ) REFERENCES Catagory( id )
);
]]></string>
</script>
</layout>
</project>

148
db/LegoLog Schema Proj.dbs.bak
View File

@@ -1,19 +1,143 @@
<?xml version="1.0" encoding="UTF-8" ?>
<project name="AppEng 2021/22" id="Project-1b11" database="LogicalDesign" >
<schema name="AppEng 2021/22" >
<table name="LegoBrick" >
<column name="id" type="varchar" length="16" jt="12" mandatory="y" />
<column name="name" type="varchar" length="100" jt="12" mandatory="y" />
<index name="pk_LegoBrick" unique="PRIMARY_KEY" >
<project name="schema.sql" id="Project-323" database="Sqlite" >
<schema name="public" >
<table name="Brick" spec="" >
<column name="id" type="varchar" length="20" jt="12" mandatory="y" />
<column name="name" type="text" length="100" jt="-1" mandatory="y" />
<column name="colour" type="integer" jt="4" />
<column name="catagory" type="integer" jt="4" />
<column name="weight" type="decimal" jt="3" />
<column name="dimensions_x" type="integer" jt="4" />
<column name="dimensions_y" type="integer" jt="4" />
<column name="dimensions_z" type="integer" jt="4" />
<column name="date_from" type="date" jt="91" />
<column name="date_to" type="date" jt="91" />
<index name="pk_Brick" unique="PRIMARY_KEY" >
<column name="id" />
</index>
<fk name="FK_Brick BrickColour" to_schema="public" to_table="BrickColour" options="" >
<fk_column name="colour" pk="id" />
</fk>
<fk name="FK_Brick Catagory" to_schema="public" to_table="Catagory" options="" >
<fk_column name="catagory" pk="id" />
</fk>
</table>
<table name="BrickColour" spec="" >
<column name="id" type="integer" jt="4" mandatory="y" />
<column name="name" type="varchar" length="100" jt="12" />
<column name="hexrgb" type="varchar" length="6" jt="12" mandatory="y" />
<column name="col_type" type="integer" jt="4" />
<index name="pk_BrickColour" unique="PRIMARY_KEY" >
<column name="id" />
</index>
<fk name="FK_BrickColour ColourType" to_schema="public" to_table="ColourType" options="" >
<fk_column name="col_type" pk="id" />
</fk>
</table>
<table name="BrickSet" spec="" >
<column name="id" type="varchar" length="20" jt="12" mandatory="y" />
<column name="catagory" type="integer" jt="4" />
<column name="name" type="varchar" length="100" jt="12" />
<column name="date_released" type="date" jt="91" />
<column name="dimensions_x" type="decimal" jt="3" />
<column name="dimensions_y" type="decimal" jt="3" />
<column name="dimensions_z" type="decimal" jt="3" />
<fk name="FK_BrickSet Catagory" to_schema="public" to_table="Catagory" options="" >
<fk_column name="catagory" pk="id" />
</fk>
</table>
<table name="Catagory" spec="" >
<column name="id" type="integer" jt="4" mandatory="y" />
<column name="name" type="varchar" length="100" jt="12" />
<index name="pk_Catagory" unique="PRIMARY_KEY" >
<column name="id" />
</index>
</table>
<table name="ColourType" spec="" >
<column name="id" type="integer" jt="4" mandatory="y" />
<column name="name" type="varchar" length="64" jt="12" />
<index name="pk_ColourType" unique="PRIMARY_KEY" >
<column name="id" />
</index>
</table>
<table name="SetBrick" spec="" >
<column name="id" type="varchar" length="50" jt="12" mandatory="y" />
<column name="set_id" type="varchar" length="50" jt="12" mandatory="y" />
<column name="brick_id" type="varchar" length="50" jt="12" mandatory="y" />
<column name="amount" type="integer" jt="4" />
<index name="pk_SetBrick" unique="PRIMARY_KEY" >
<column name="id" />
</index>
<fk name="FK_SetBrick BrickSet" to_schema="public" to_table="BrickSet" options="" >
<fk_column name="set_id" pk="id" />
</fk>
<fk name="FK_SetBrick Brick" to_schema="public" to_table="Brick" options="" >
<fk_column name="brick_id" pk="id" />
</fk>
</table>
</schema>
<layout name="Default Layout" id="Layout-478" show_relation="columns" >
<entity schema="AppEng 2021/22" name="LegoBrick" color="C1D8EE" x="48" y="80" />
<callout x="752" y="48" pointer="Round" >
<comment><![CDATA[Right-click the layout to create entities or shapes, create relations using drag and drop.
Save the design to file. Use the Convert menu option to convert the model to a physical model.]]></comment>
</callout>
<layout name="Layout" id="Layout-20e8" show_relation="columns" >
<entity schema="public" name="Brick" color="BED3F4" x="272" y="48" />
<entity schema="public" name="BrickColour" color="BED3F4" x="464" y="48" />
<entity schema="public" name="BrickSet" color="BED3F4" x="272" y="352" />
<entity schema="public" name="Catagory" color="BED3F4" x="464" y="256" />
<entity schema="public" name="ColourType" color="BED3F4" x="624" y="48" />
<entity schema="public" name="SetBrick" color="BED3F4" x="48" y="256" />
<script name="schema.sql" id="Editor-1de1" language="SQL" >
<string><![CDATA[CREATE TABLE Catagory (
id integer NOT NULL PRIMARY KEY ,
name varchar(100)
);
CREATE TABLE ColourType (
id integer NOT NULL PRIMARY KEY ,
name varchar(64)
);
CREATE TABLE BrickSet (
id varchar(20) NOT NULL ,
catagory integer ,
name varchar(100) ,
date_released date ,
dimensions_x decimal ,
dimensions_y decimal ,
dimensions_z decimal ,
FOREIGN KEY ( catagory ) REFERENCES Catagory( id )
);
CREATE TABLE SetBrick (
id varchar(50) NOT NULL PRIMARY KEY ,
set_id varchar(50) NOT NULL ,
brick_id varchar(50) NOT NULL ,
amount integer ,
FOREIGN KEY ( set_id ) REFERENCES BrickSet( id ) ,
FOREIGN KEY ( brick_id ) REFERENCES Brick( id ) ,
);
CREATE TABLE BrickColour (
id integer NOT NULL PRIMARY KEY ,
name varchar(100) ,
hexrgb varchar(6) NOT NULL ,
col_type integer ,
FOREIGN KEY ( col_type ) REFERENCES ColourType( id )
);
CREATE TABLE Brick (
id varchar(20) NOT NULL PRIMARY KEY ,
name text(100) NOT NULL ,
colour integer ,
catagory integer ,
weight decimal ,
dimensions_x integer ,
dimensions_y integer ,
dimensions_z integer ,
date_from date ,
date_to date ,
FOREIGN KEY ( colour ) REFERENCES BrickColour( id ) ,
FOREIGN KEY ( catagory ) REFERENCES Catagory( id )
);
]]></string>
</script>
</layout>
</project>

BIN
db/dev.sqlite Normal file
View File

Binary file not shown.

1101
db/error.txt Normal file
View File

File diff suppressed because it is too large Load Diff

59
db/imagestealer.js
View File

@@ -3,7 +3,62 @@
// download image from bricklink for every item in the tab-delimited database Parts.txt
const fs = require('fs');
const axios = require('axios');
// For sets make sets.txt
const parts = fs.readFileSync('res/Sets.txt', 'utf8').toString().split('\n').map((i) => i.split('\t'));
let parts = fs.readFileSync('res/Parts.txt', 'utf8').toString().split('\n').map((i) => i.split('\t'));
async function timeout(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
console.log(parts);
async function downloadImage(url, filename) {
try {
const res = await axios.get(url, {
method: 'GET',
headers: { 'User-Agent':'Chrome/96.0.4664.175' } ,
responseType: 'stream'
});
if (!res.data) {
console.log(`${filename} failed to start downloading`);
fs.appendFileSync('error.txt', `${url}\n`);
return;
}
return new Promise((resolve, reject) => {
res.data.pipe(fs.createWriteStream(filename));
res.data.on('end', () => {
console.log('downloaded file ' + filename);
fs.appendFileSync('saved.txt', `${url}\n`);
resolve();
})
res.data.on('error', () => {
console.log('error downloading file ' + filename);
fs.appendFileSync('error.txt', `${url}\n`);
resolve();
})
});
} catch (e) {
console.log(`${filename} failed to start downloading`);
fs.appendFileSync('error.txt', `${url}\n`);
return;
}
}
async function main() {
for (let i = 0; i < parts.length; i++) {
const part = parts[i];
// for sets use https://img.bricklink.com/ItemImage/SL/${part[2]}.png
// for for pieces use https://img.bricklink.com/ItemImage/PL/${part[2]}.png
const url = `https://img.bricklink.com/ItemImage/SL/${part[2]}.png`;
const filename = `res/image/${part[2]}.png`;
await downloadImage(url, filename);
// await timeout(10); // let's not get rate limited
console.log(`${i}/${parts.length} ${url}`);
}
}
main();

1
db/res/.gitattributes vendored Normal file
View File

@@ -0,0 +1 @@
*.tar.gz filter=lfs diff=lfs merge=lfs -text

3
db/res/Parts.txt
View File

@@ -1,6 +1,3 @@
Category ID Category Name Number Name Weight (in Grams) Dimensions
143 (Other) merge Catalog Entry for Merging Parts ? ? x ? x ?
85 Homemaker 1 Homemaker Bookcase 2 x 4 x 4 5.8 2 x 4 x 4
2 Baseplate 10 Baseplate 24 x 32 (Undetermined Type) 74 24 x 32 x 0

3
db/res/Sets.txt
View File

@@ -1,6 +1,3 @@
Category ID Category Name Number Name Year Released Weight (in Grams) Dimensions
67 Town / Classic Town / Supplemental 0011-2 Town Mini-Figures 1982 ? ? x ? x ?
34 Space / Supplemental 0012-1 Space Mini-Figures 1982 ? ? x ? x ?
34 Space / Supplemental 0013-1 Space Mini-Figures 1982 ? ? x ? x ?

1
db/res/sets.json Normal file
View File

File diff suppressed because one or more lines are too long

49
db/schema.sql
View File

@@ -1,3 +1,11 @@
CREATE TABLE LegoBrickInventory (
id varchar(50)
);
CREATE TABLE LegoSetInventory (
id varchar(50)
);
CREATE TABLE Catagory (
id integer NOT NULL PRIMARY KEY ,
name varchar(100)
@@ -5,11 +13,19 @@ CREATE TABLE Catagory (
CREATE TABLE ColourType (
id integer NOT NULL PRIMARY KEY ,
"type" varchar(64)
name varchar(64)
);
CREATE TABLE "Set" (
id varchar(20) NOT NULL ,
CREATE TABLE LegoBrickColour (
id integer NOT NULL PRIMARY KEY ,
name varchar(100) ,
hexrgb varchar(6) NOT NULL ,
col_type integer ,
FOREIGN KEY ( col_type ) REFERENCES ColourType( id )
);
CREATE TABLE LegoSet (
id varchar(50) NOT NULL PRIMARY KEY ,
catagory integer ,
name varchar(100) ,
date_released date ,
@@ -19,18 +35,8 @@ CREATE TABLE "Set" (
FOREIGN KEY ( catagory ) REFERENCES Catagory( id )
);
CREATE TABLE BrickColour (
id integer NOT NULL PRIMARY KEY ,
name varchar(100) ,
hexrgb varchar(6) NOT NULL ,
"type" integer ,
date_from date ,
date_to date ,
FOREIGN KEY ( "type" ) REFERENCES ColourType( id )
);
CREATE TABLE Brick (
id varchar(20) NOT NULL PRIMARY KEY ,
CREATE TABLE LegoBrick (
id varchar(50) NOT NULL PRIMARY KEY ,
name text(100) NOT NULL ,
colour integer ,
catagory integer ,
@@ -38,7 +44,18 @@ CREATE TABLE Brick (
dimensions_x integer ,
dimensions_y integer ,
dimensions_z integer ,
FOREIGN KEY ( colour ) REFERENCES BrickColour( id ) ,
date_from date ,
date_to date ,
FOREIGN KEY ( colour ) REFERENCES LegoBrickColour( id ) ,
FOREIGN KEY ( catagory ) REFERENCES Catagory( id )
);
CREATE TABLE SetDescriptor (
id varchar(50) NOT NULL PRIMARY KEY ,
set_id varchar(50) NOT NULL ,
brick_id varchar(50) NOT NULL ,
amount integer ,
FOREIGN KEY ( set_id ) REFERENCES LegoSet( id ) ,
FOREIGN KEY ( brick_id ) REFERENCES LegoBrick( id )
);

52
db/setstealer.js Normal file
View File

@@ -0,0 +1,52 @@
// scrapes bricklink for the every piece and amounts in a set of lego
const fs = require('fs');
const axios = require('axios');
// For sets make sets.txt
const sets = fs.readFileSync('res/Sets.txt', 'utf8').toString().split('\n').map((i) => i.split('\t'));
// output format:
// setid: { pieceid: amount, pieceid: amount, ... }
async function post(url) {
// axios return HTML from website
try {
const res = await axios.get(url, {
method: 'POST',
headers: { 'User-Agent':'Chrome/96.0.4664.175' } ,
});
return res.data.toString();
} catch(e) {
fs.appendFileSync('error-set.txt', `${url}\n`);
console.log(`Failed to download ${url}`);
return;
}
}
async function main() {
// sometimes fails on minifigures - doesn't matter though, it's correct enough
const regex = /class=".*?IV_ITEM".*?if \(brickList\["(.*?)"]\).*?nbsp;(.*?)&nbsp;/g
const output = {}
for (let i = 0; i < sets.length; i++) {
const set = sets[i];
const data = await post(`https://www.bricklink.com/catalogItemInv.asp?S=${set[2]}`);
output[set[2]] = {};
let pieceCount = 0;
let m;
while ((m = regex.exec(data)) !== null) {
if (m.index === regex.lastIndex) {
regex.lastIndex++;
}
pieceCount += parseInt(m[2]);
output[set[2]] = { ...output[set[2]], [m[1]]: parseInt(m[2]) };
}
console.log(`${i}/${sets.length} ${set[2]} has ${pieceCount} pieces`);
fs.writeFileSync('res/sets.json', JSON.stringify(output));
}
}
main();