Skip to content

Commit

Permalink
fix2 block
Browse files Browse the repository at this point in the history
  • Loading branch information
@killua-eu
killua-eu committed Jan 14, 2024
1 parent 4926e38 commit 402bd63
Show file tree
Hide file tree
Showing 3 changed files with 203 additions and 101 deletions.
38 changes: 38 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,48 @@

Stor is a content addressable storage microservice.

## Setup

- Configure stor devices and dgs and the notification system by adding `stor.yaml` and `notify.yaml` to `/var/www/html/data/glued-stor/config`
- Create first bucket

```shell
curl 'https://glued/api/stor/v1/buckets' --compressed -X POST -H 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Upgrade-Insecure-Requests: 1' -H 'Sec-Fetch-Dest: document' -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-Site: cross-site' -H 'Content-Type: application/json' -H 'Origin: null' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' --data-raw '{"name": "Industra bucket", "dgs": ["66bf39ed-3913-4985-a775-ef3c87cfaee4"] }'
```

```shell
curl -k -X POST https://glued.industra.space/api/stor/v1/buckets/ade1a7f2-3ac4-4e1e-b4bb-96d0f21ada7b/objects -H 'Authorization: Bearer gtk_35MCHFgkNh1PymQEOLStzMtESdo4DZXykoYWvjX9QcQ=' -H 'Content-Type: multipart/form-data' -F 'links={ "Ryanair.pdf": { "uuid": "77384ad2-80a4-11ee-9edc-9747c96a1231", "parent": "bd4a3c4a-80a6-11ee-8e34-67217a8a9f66", "app": { "name": "client-name", "instance": "https://glued.example.com", "discover": "/api/client-name/v1/endpoint/someID" }}, "eastern loves_INDUSTRA.png": {}};type=application/json' -F 'file[]=@/home/killua/Ryanair.pdf' -F 'file[]=@/home/killua/eastern loves_INDUSTRA.png' -F 'file[]=@/home/killua/todo-petru' -F 'field1=fiels2' | jq .
```
TODO: token

## Concepts

## Using from the command line

```bash
#new
curl -k -X POST https://glued/api/stor/v1/buckets/1fcb4d5c-5364-4cf3-b24e-070c4c71f8d2/objects -H 'Authorization: Bearer gtk_35MCHFgkNh1PymQEOLStzMtESdo4DZXykoYWvjX9QcQ=' -H 'Content-Type: multipart/form-data' -F 'refs={"Ryanair.pdf":{"refs":{"predecessor":"77384ad2-80a4-11ee-9edc-9747c96a1231","sekatko:vpd":"bd4a3c4a-80a6-11ee-8e34-67217a8a9f66"},"meta":{"sekato":{"ocr":"sometext"},"otherapp":{"whatever":"https://glued.example.com"}}}};type=application/json' -F 'file[]=@/home/killua/Ryanair.pdf' -F 'file[]=@/home/killua/eastern loves_INDUSTRA.png' -F 'file[]=@/home/killua/todo-petru' -F 'field1=fiels2' | jq .


{
"Ryanair.pdf": {
"refs": {
"predecessor": "77384ad2-80a4-11ee-9edc-9747c96a1231",
"sekatko:vpd": "bd4a3c4a-80a6-11ee-8e34-67217a8a9f66"
},
"meta": {
"sekato": {
"ocr": "sometext"
},
"otherapp": {
"whatever": "https://glued.example.com"
}
}
}
}

```

```bash
curl -k -X POST https://glued/api/stor/files/v1 \
-H 'Authorization: Bearer gtk_35MCHFgkNh1PymQEOLStzMtESdo4DZXykoYWvjX9QcQ=' \
Expand Down
145 changes: 71 additions & 74 deletions glued/Config/Migrations/20210823131604_stor.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,20 @@ SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO';
SET NAMES utf8mb4;

DROP TABLE IF EXISTS `t_stor_bucket_dgs`;
CREATE TABLE `t_stor_bucket_dgs` (
`bucket` binary(16) NOT NULL COMMENT 'Bucket UUID (foreign key)',
`dg` binary(16) NOT NULL COMMENT 'Bucket device group UUID (foreign key)',
UNIQUE KEY `unique_bucket_dg` (`bucket`,`dg`),
KEY `bucket_uuid` (`bucket`),
KEY `dg` (`dg`),
CONSTRAINT `t_stor_bucket_dgs_ibfk_1` FOREIGN KEY (`bucket`) REFERENCES `t_stor_buckets` (`uuid`) ON DELETE RESTRICT ON UPDATE CASCADE,
CONSTRAINT `t_stor_bucket_dgs_ibfk_2` FOREIGN KEY (`dg`) REFERENCES `t_stor_dgs` (`uuid`) ON DELETE RESTRICT ON UPDATE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor Buckets are isolated containers for links with assignable storage device groups (dgs)';

DROP TABLE IF EXISTS `t_stor_dgs`;
DROP TABLE IF EXISTS `t_stor_files`;
DROP TABLE IF EXISTS `t_stor_objects_meta`;

DROP TABLE IF EXISTS `t_stor_buckets`;
DROP TABLE IF EXISTS `t_stor_configlog`;
DROP TABLE IF EXISTS `t_stor_devices`;
DROP TABLE IF EXISTS `t_stor_objects_refs`;

DROP TABLE IF EXISTS `t_stor_objects`;


DROP TABLE IF EXISTS `t_stor_objects_replicas`;

CREATE TABLE `t_stor_buckets` (
`uuid` binary(16) NOT NULL DEFAULT (uuid_to_bin(uuid(),true)) COMMENT 'Bucket UUID',
`data` json NOT NULL COMMENT 'JSON data',
Expand All @@ -28,7 +30,7 @@ CREATE TABLE `t_stor_buckets` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor Buckets are isolated containers for links with assignable storage device groups (dgs)';


DROP TABLE IF EXISTS `t_stor_configlog`;

CREATE TABLE `t_stor_configlog` (
`uuid` binary(16) NOT NULL COMMENT 'Device / device group UUID.',
`data` json NOT NULL COMMENT 'Config JSON.',
Expand All @@ -38,15 +40,15 @@ CREATE TABLE `t_stor_configlog` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor devices/dgs config log';


DROP TABLE IF EXISTS `t_stor_devices`;

CREATE TABLE `t_stor_devices` (
`uuid` binary(16) NOT NULL DEFAULT (uuid_to_bin(uuid(),true)) COMMENT 'Device group UUID',
`data` json NOT NULL COMMENT 'Device group JSON',
PRIMARY KEY (`uuid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor devices.';


DROP TABLE IF EXISTS `t_stor_dgs`;

CREATE TABLE `t_stor_dgs` (
`uuid` binary(16) NOT NULL DEFAULT (uuid_to_bin(uuid(),true)) COMMENT 'Device group UUID',
`data` json NOT NULL COMMENT 'Device group JSON',
Expand All @@ -56,84 +58,79 @@ CREATE TABLE `t_stor_dgs` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor Buckets are isolated containers for links with assignable storage device groups (dgs)';


DROP TABLE IF EXISTS `t_stor_links`;
CREATE TABLE `t_stor_links` (
`uuid` binary(16) NOT NULL COMMENT 'Link UUID.',
`bucket` binary(16) NOT NULL COMMENT 'Bucket UUID.',
`object` binary(16) NOT NULL COMMENT 'Data object UUID.',
`name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'file' COMMENT 'Default filename.',
PRIMARY KEY (`uuid`),
KEY `bucket` (`bucket`),
KEY `name` (`name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci ROW_FORMAT=DYNAMIC COMMENT='Content aware storage links table. Entries in t_stor_objects are unique, t_stor_links provides links to appropriate locations with a user-friendly name.';


DROP TABLE IF EXISTS `t_stor_links_meta`;
CREATE TABLE `t_stor_links_meta` (
`uuid` binary(16) NOT NULL COMMENT 'Link UUID',
`namespace` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT 'Metadata namespace',
`data` json NOT NULL COMMENT 'Metadata json',
PRIMARY KEY (`uuid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='App specific metadata related to a particular link';

CREATE TABLE `t_stor_files` (
`c_hash` binary(64) GENERATED ALWAYS AS (unhex(json_unquote(json_extract(`c_data`,_utf8mb4'$.hash."sha3-512"')))) STORED NOT NULL COMMENT 'File main hash (sha3-512)',
`c_data` json NOT NULL COMMENT 'File metadata',
`c_size` bigint GENERATED ALWAYS AS (cast(json_unquote(json_extract(`c_data`,_utf8mb4'$.size')) as unsigned)) STORED COMMENT 'Filesize',
`c_mime` varchar(80) GENERATED ALWAYS AS (json_unquote(json_extract(`c_data`,_utf8mb4'$.mime.type'))) STORED COMMENT 'MIME type',
`c_ext` varchar(32) GENERATED ALWAYS AS (json_unquote(json_extract(`c_data`,_utf8mb4'$.mime.ext'))) STORED COMMENT 'MIME extension',
`c_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Timestamp object created',
PRIMARY KEY (`c_hash`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci ROW_FORMAT=DYNAMIC COMMENT='Content addressable storage file metadata.';

DROP TABLE IF EXISTS `t_stor_links_rels`;
CREATE TABLE `t_stor_links_rels` (
`uuid1` binary(16) NOT NULL COMMENT 'Link UUID 1',
`uuid2` binary(16) NOT NULL COMMENT 'Link UUID 2',
`type` varchar(64) NOT NULL COMMENT 'Relationship type',
`ts_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create timestamp',
KEY `uuid1` (`uuid1`),
KEY `uuid2` (`uuid2`),
KEY `type` (`type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Relationships between links';


DROP TABLE IF EXISTS `t_stor_objects`;
CREATE TABLE `t_stor_objects` (
`c_uuid` binary(16) GENERATED ALWAYS AS (uuid_to_bin(json_unquote(json_extract(`c_data`,_utf8mb4'$.uuid')),true)) STORED NOT NULL,
`c_data` json NOT NULL COMMENT 'Object metadata',
`c_size` bigint GENERATED ALWAYS AS (cast(json_unquote(json_extract(`c_data`,_utf8mb4'$.size')) as unsigned)) STORED COMMENT 'Object size',
`c_mime` varchar(80) GENERATED ALWAYS AS (json_unquote(json_extract(`c_data`,_utf8mb4'$.mime.type'))) STORED COMMENT 'Object MIME type',
`c_ext` varchar(32) GENERATED ALWAYS AS (json_unquote(json_extract(`c_data`,_utf8mb4'$.mime.ext'))) STORED COMMENT 'Object extension',
`c_seekhash` binary(16) GENERATED ALWAYS AS (unhex(json_unquote(json_extract(`c_data`,_utf8mb4'$.hash.md5')))) STORED COMMENT 'Indexed / seekable hash (md5)',
`c_mainhash` binary(64) GENERATED ALWAYS AS (unhex(json_unquote(json_extract(`c_data`,_utf8mb4'$.hash.sha3-512')))) STORED COMMENT 'Main hash (sha3-512)',
`c_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Timestamp object created',
PRIMARY KEY (`c_uuid`),
UNIQUE KEY `unique_mainhash` (`c_mainhash`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci ROW_FORMAT=DYNAMIC COMMENT='Content addressable storage data objects.';
`bucket` binary(16) NOT NULL COMMENT 'Bucket UUID',
`object` binary(16) NOT NULL COMMENT 'Object UUID of the parent bucket',
`hash` binary(64) NOT NULL COMMENT 'Object hash',
`name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'Object filename (can differ from bucket to bucket)',
`ts_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Timestamp from when object was created in the bucket.',
UNIQUE KEY `unique_bucket_object` (`bucket`,`object`),
KEY `object` (`object`),
KEY `bucket` (`bucket`),
KEY `hash` (`hash`(12))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor objects are files placed into isolated containers (Buckets)';



CREATE TABLE `t_stor_objects_meta` (
`uuid` binary(16) NOT NULL COMMENT 'Link UUID',
`namespace` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'Metadata namespace',
`data` json NOT NULL COMMENT 'Metadata json',
PRIMARY KEY (`uuid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='App specific metadata related to a particular link';



CREATE TABLE `t_stor_objects_refs` (
`uuid1` binary(16) NOT NULL COMMENT 'Link UUID 1',
`uuid2` binary(16) NOT NULL COMMENT 'Link UUID 2',
`scope` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT 'Reference scope',
`kind` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT 'Reference kind',
`ts_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create timestamp',
KEY `uuid1` (`uuid1`),
KEY `uuid2` (`uuid2`),
KEY `scope` (`scope`),
KEY `kind` (`kind`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='References between objects';



DROP TABLE IF EXISTS `t_stor_objects_replicas`;
CREATE TABLE `t_stor_objects_replicas` (
`object` binary(16) NOT NULL COMMENT 'Object UUID',
`device` binary(16) NOT NULL COMMENT 'Device UUID',
`ts_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Timestamp from when object is scheduled for balancing onto device.',
`ts_updated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Timestamp from when object was last tested/modified on its balanced/consistent status',
`ts_locked` timestamp NULL DEFAULT NULL COMMENT 'Timestamp until when object is locked for other operations on device.',
`balanced` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Object is present on devices',
`consistent` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Object file size and hash on device matches that which is stored in the database.'
`consistent` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Object file size and hash on device matches that which is stored in the database.',
UNIQUE KEY `unique_object_device` (`object`,`device`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;

CREATE TABLE `t_stor_bucket_dgs` (
`bucket` binary(16) NOT NULL COMMENT 'Bucket UUID (foreign key)',
`dg` binary(16) NOT NULL COMMENT 'Bucket device group UUID (foreign key)',
UNIQUE KEY `unique_bucket_dg` (`bucket`,`dg`),
KEY `bucket_uuid` (`bucket`),
KEY `dg` (`dg`),
CONSTRAINT `t_stor_bucket_dgs_ibfk_1` FOREIGN KEY (`bucket`) REFERENCES `t_stor_buckets` (`uuid`) ON DELETE RESTRICT ON UPDATE CASCADE,
CONSTRAINT `t_stor_bucket_dgs_ibfk_2` FOREIGN KEY (`dg`) REFERENCES `t_stor_dgs` (`uuid`) ON DELETE RESTRICT ON UPDATE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor Buckets are isolated containers for links with assignable storage device groups (dgs)';



DROP TABLE IF EXISTS `t_stor_statuslog`;
CREATE TABLE `t_stor_statuslog` (
`uuid` binary(16) NOT NULL DEFAULT (uuid_to_bin(uuid(),true)) COMMENT 'Status line UUID',
`uuid_dg` binary(16) NOT NULL COMMENT 'Dg UUID',
`uuid_dev` binary(16) NOT NULL COMMENT 'Device UUID',
`prio` int NOT NULL DEFAULT '1000' COMMENT 'Device priority in dg',
`role` varchar(16) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL DEFAULT 'undefined' COMMENT 'Device role dg',
`data` json NOT NULL COMMENT 'Device status JSON',
`health` varchar(16) GENERATED ALWAYS AS (coalesce(json_unquote(json_extract(`data`,_utf8mb4'$.health')),_utf8mb4'undefined')) VIRTUAL COMMENT 'Device health status',
`ts_created` datetime(1) DEFAULT CURRENT_TIMESTAMP(1) COMMENT 'Timestamp Created with precision of 1 second',
`ts_updated` datetime(1) DEFAULT CURRENT_TIMESTAMP(1) ON UPDATE CURRENT_TIMESTAMP(1) COMMENT 'Timestamp Updated with precision of 1 second',
`boundary` bigint GENERATED ALWAYS AS ((floor((`ts_created` / 86400)) * 86400)) STORED COMMENT 'Boundary timestamp rounded to 24 hours (86400 seconds)',
`data_hash` binary(16) GENERATED ALWAYS AS (unhex(md5(`data`))) STORED COMMENT 'Device status hash',
`uniq_hash` binary(16) GENERATED ALWAYS AS (unhex(md5(concat(`uuid_dg`,`uuid_dev`,`data`,`boundary`)))) STORED COMMENT 'Unique row hash',
PRIMARY KEY (`uuid`),
UNIQUE KEY `uk_uniq_hash` (`uniq_hash`),
KEY `idx_health_prefix` (`health`(2))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor devices/dgs status log. Uniqueness based on Device UUID, partial hash, and creation timestamp (24 hours intervals)';

/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
Expand Down
Loading

0 comments on commit 402bd63

Please sign in to comment.