diff --git a/README.md b/README.md index a1fae01..9963f3c 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,48 @@ Stor is a content addressable storage microservice. +## Setup + +- Configure stor devices and dgs and the notification system by adding `stor.yaml` and `notify.yaml` to `/var/www/html/data/glued-stor/config` +- Create first bucket + +```shell +curl 'https://glued/api/stor/v1/buckets' --compressed -X POST -H 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' -H 'Accept-Language: en-US,en;q=0.5' -H 'Accept-Encoding: gzip, deflate, br' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Upgrade-Insecure-Requests: 1' -H 'Sec-Fetch-Dest: document' -H 'Sec-Fetch-Mode: navigate' -H 'Sec-Fetch-Site: cross-site' -H 'Content-Type: application/json' -H 'Origin: null' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' --data-raw '{"name": "Industra bucket", "dgs": ["66bf39ed-3913-4985-a775-ef3c87cfaee4"] }' +``` + +```shell +curl -k -X POST https://glued.industra.space/api/stor/v1/buckets/ade1a7f2-3ac4-4e1e-b4bb-96d0f21ada7b/objects -H 'Authorization: Bearer gtk_35MCHFgkNh1PymQEOLStzMtESdo4DZXykoYWvjX9QcQ=' -H 'Content-Type: multipart/form-data' -F 'links={ "Ryanair.pdf": { "uuid": "77384ad2-80a4-11ee-9edc-9747c96a1231", "parent": "bd4a3c4a-80a6-11ee-8e34-67217a8a9f66", "app": { "name": "client-name", "instance": "https://glued.example.com", "discover": "/api/client-name/v1/endpoint/someID" }}, "eastern loves_INDUSTRA.png": {}};type=application/json' -F 'file[]=@/home/killua/Ryanair.pdf' -F 'file[]=@/home/killua/eastern loves_INDUSTRA.png' -F 'file[]=@/home/killua/todo-petru' -F 'field1=fiels2' | jq . +``` +TODO: token + ## Concepts ## Using from the command line +```bash +#new +curl -k -X POST https://glued/api/stor/v1/buckets/1fcb4d5c-5364-4cf3-b24e-070c4c71f8d2/objects -H 'Authorization: Bearer gtk_35MCHFgkNh1PymQEOLStzMtESdo4DZXykoYWvjX9QcQ=' -H 'Content-Type: multipart/form-data' -F 'refs={"Ryanair.pdf":{"refs":{"predecessor":"77384ad2-80a4-11ee-9edc-9747c96a1231","sekatko:vpd":"bd4a3c4a-80a6-11ee-8e34-67217a8a9f66"},"meta":{"sekato":{"ocr":"sometext"},"otherapp":{"whatever":"https://glued.example.com"}}}};type=application/json' -F 'file[]=@/home/killua/Ryanair.pdf' -F 'file[]=@/home/killua/eastern loves_INDUSTRA.png' -F 'file[]=@/home/killua/todo-petru' -F 'field1=fiels2' | jq . + + + { + "Ryanair.pdf": { + "refs": { + "predecessor": "77384ad2-80a4-11ee-9edc-9747c96a1231", + "sekatko:vpd": "bd4a3c4a-80a6-11ee-8e34-67217a8a9f66" + }, + "meta": { + "sekato": { + "ocr": "sometext" + }, + "otherapp": { + "whatever": "https://glued.example.com" + } + } + } +} + +``` + ```bash curl -k -X POST https://glued/api/stor/files/v1 \ -H 'Authorization: Bearer gtk_35MCHFgkNh1PymQEOLStzMtESdo4DZXykoYWvjX9QcQ=' \ diff --git a/glued/Config/Migrations/20210823131604_stor.sql b/glued/Config/Migrations/20210823131604_stor.sql index 0ccf821..cb5dea1 100644 --- a/glued/Config/Migrations/20210823131604_stor.sql +++ b/glued/Config/Migrations/20210823131604_stor.sql @@ -8,18 +8,20 @@ SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; SET NAMES utf8mb4; DROP TABLE IF EXISTS `t_stor_bucket_dgs`; -CREATE TABLE `t_stor_bucket_dgs` ( - `bucket` binary(16) NOT NULL COMMENT 'Bucket UUID (foreign key)', - `dg` binary(16) NOT NULL COMMENT 'Bucket device group UUID (foreign key)', - UNIQUE KEY `unique_bucket_dg` (`bucket`,`dg`), - KEY `bucket_uuid` (`bucket`), - KEY `dg` (`dg`), - CONSTRAINT `t_stor_bucket_dgs_ibfk_1` FOREIGN KEY (`bucket`) REFERENCES `t_stor_buckets` (`uuid`) ON DELETE RESTRICT ON UPDATE CASCADE, - CONSTRAINT `t_stor_bucket_dgs_ibfk_2` FOREIGN KEY (`dg`) REFERENCES `t_stor_dgs` (`uuid`) ON DELETE RESTRICT ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor Buckets are isolated containers for links with assignable storage device groups (dgs)'; - +DROP TABLE IF EXISTS `t_stor_dgs`; +DROP TABLE IF EXISTS `t_stor_files`; +DROP TABLE IF EXISTS `t_stor_objects_meta`; DROP TABLE IF EXISTS `t_stor_buckets`; +DROP TABLE IF EXISTS `t_stor_configlog`; +DROP TABLE IF EXISTS `t_stor_devices`; +DROP TABLE IF EXISTS `t_stor_objects_refs`; + +DROP TABLE IF EXISTS `t_stor_objects`; + + +DROP TABLE IF EXISTS `t_stor_objects_replicas`; + CREATE TABLE `t_stor_buckets` ( `uuid` binary(16) NOT NULL DEFAULT (uuid_to_bin(uuid(),true)) COMMENT 'Bucket UUID', `data` json NOT NULL COMMENT 'JSON data', @@ -28,7 +30,7 @@ CREATE TABLE `t_stor_buckets` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor Buckets are isolated containers for links with assignable storage device groups (dgs)'; -DROP TABLE IF EXISTS `t_stor_configlog`; + CREATE TABLE `t_stor_configlog` ( `uuid` binary(16) NOT NULL COMMENT 'Device / device group UUID.', `data` json NOT NULL COMMENT 'Config JSON.', @@ -38,7 +40,7 @@ CREATE TABLE `t_stor_configlog` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor devices/dgs config log'; -DROP TABLE IF EXISTS `t_stor_devices`; + CREATE TABLE `t_stor_devices` ( `uuid` binary(16) NOT NULL DEFAULT (uuid_to_bin(uuid(),true)) COMMENT 'Device group UUID', `data` json NOT NULL COMMENT 'Device group JSON', @@ -46,7 +48,7 @@ CREATE TABLE `t_stor_devices` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor devices.'; -DROP TABLE IF EXISTS `t_stor_dgs`; + CREATE TABLE `t_stor_dgs` ( `uuid` binary(16) NOT NULL DEFAULT (uuid_to_bin(uuid(),true)) COMMENT 'Device group UUID', `data` json NOT NULL COMMENT 'Device group JSON', @@ -56,55 +58,56 @@ CREATE TABLE `t_stor_dgs` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor Buckets are isolated containers for links with assignable storage device groups (dgs)'; -DROP TABLE IF EXISTS `t_stor_links`; -CREATE TABLE `t_stor_links` ( - `uuid` binary(16) NOT NULL COMMENT 'Link UUID.', - `bucket` binary(16) NOT NULL COMMENT 'Bucket UUID.', - `object` binary(16) NOT NULL COMMENT 'Data object UUID.', - `name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'file' COMMENT 'Default filename.', - PRIMARY KEY (`uuid`), - KEY `bucket` (`bucket`), - KEY `name` (`name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci ROW_FORMAT=DYNAMIC COMMENT='Content aware storage links table. Entries in t_stor_objects are unique, t_stor_links provides links to appropriate locations with a user-friendly name.'; - - -DROP TABLE IF EXISTS `t_stor_links_meta`; -CREATE TABLE `t_stor_links_meta` ( - `uuid` binary(16) NOT NULL COMMENT 'Link UUID', - `namespace` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT 'Metadata namespace', - `data` json NOT NULL COMMENT 'Metadata json', - PRIMARY KEY (`uuid`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='App specific metadata related to a particular link'; +CREATE TABLE `t_stor_files` ( + `c_hash` binary(64) GENERATED ALWAYS AS (unhex(json_unquote(json_extract(`c_data`,_utf8mb4'$.hash."sha3-512"')))) STORED NOT NULL COMMENT 'File main hash (sha3-512)', + `c_data` json NOT NULL COMMENT 'File metadata', + `c_size` bigint GENERATED ALWAYS AS (cast(json_unquote(json_extract(`c_data`,_utf8mb4'$.size')) as unsigned)) STORED COMMENT 'Filesize', + `c_mime` varchar(80) GENERATED ALWAYS AS (json_unquote(json_extract(`c_data`,_utf8mb4'$.mime.type'))) STORED COMMENT 'MIME type', + `c_ext` varchar(32) GENERATED ALWAYS AS (json_unquote(json_extract(`c_data`,_utf8mb4'$.mime.ext'))) STORED COMMENT 'MIME extension', + `c_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Timestamp object created', + PRIMARY KEY (`c_hash`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci ROW_FORMAT=DYNAMIC COMMENT='Content addressable storage file metadata.'; -DROP TABLE IF EXISTS `t_stor_links_rels`; -CREATE TABLE `t_stor_links_rels` ( - `uuid1` binary(16) NOT NULL COMMENT 'Link UUID 1', - `uuid2` binary(16) NOT NULL COMMENT 'Link UUID 2', - `type` varchar(64) NOT NULL COMMENT 'Relationship type', - `ts_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create timestamp', - KEY `uuid1` (`uuid1`), - KEY `uuid2` (`uuid2`), - KEY `type` (`type`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Relationships between links'; -DROP TABLE IF EXISTS `t_stor_objects`; CREATE TABLE `t_stor_objects` ( - `c_uuid` binary(16) GENERATED ALWAYS AS (uuid_to_bin(json_unquote(json_extract(`c_data`,_utf8mb4'$.uuid')),true)) STORED NOT NULL, - `c_data` json NOT NULL COMMENT 'Object metadata', - `c_size` bigint GENERATED ALWAYS AS (cast(json_unquote(json_extract(`c_data`,_utf8mb4'$.size')) as unsigned)) STORED COMMENT 'Object size', - `c_mime` varchar(80) GENERATED ALWAYS AS (json_unquote(json_extract(`c_data`,_utf8mb4'$.mime.type'))) STORED COMMENT 'Object MIME type', - `c_ext` varchar(32) GENERATED ALWAYS AS (json_unquote(json_extract(`c_data`,_utf8mb4'$.mime.ext'))) STORED COMMENT 'Object extension', - `c_seekhash` binary(16) GENERATED ALWAYS AS (unhex(json_unquote(json_extract(`c_data`,_utf8mb4'$.hash.md5')))) STORED COMMENT 'Indexed / seekable hash (md5)', - `c_mainhash` binary(64) GENERATED ALWAYS AS (unhex(json_unquote(json_extract(`c_data`,_utf8mb4'$.hash.sha3-512')))) STORED COMMENT 'Main hash (sha3-512)', - `c_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Timestamp object created', - PRIMARY KEY (`c_uuid`), - UNIQUE KEY `unique_mainhash` (`c_mainhash`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci ROW_FORMAT=DYNAMIC COMMENT='Content addressable storage data objects.'; + `bucket` binary(16) NOT NULL COMMENT 'Bucket UUID', + `object` binary(16) NOT NULL COMMENT 'Object UUID of the parent bucket', + `hash` binary(64) NOT NULL COMMENT 'Object hash', + `name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'Object filename (can differ from bucket to bucket)', + `ts_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Timestamp from when object was created in the bucket.', + UNIQUE KEY `unique_bucket_object` (`bucket`,`object`), + KEY `object` (`object`), + KEY `bucket` (`bucket`), + KEY `hash` (`hash`(12)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor objects are files placed into isolated containers (Buckets)'; + + + +CREATE TABLE `t_stor_objects_meta` ( + `uuid` binary(16) NOT NULL COMMENT 'Link UUID', + `namespace` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'Metadata namespace', + `data` json NOT NULL COMMENT 'Metadata json', + PRIMARY KEY (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='App specific metadata related to a particular link'; + + + +CREATE TABLE `t_stor_objects_refs` ( + `uuid1` binary(16) NOT NULL COMMENT 'Link UUID 1', + `uuid2` binary(16) NOT NULL COMMENT 'Link UUID 2', + `scope` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT 'Reference scope', + `kind` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT 'Reference kind', + `ts_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Create timestamp', + KEY `uuid1` (`uuid1`), + KEY `uuid2` (`uuid2`), + KEY `scope` (`scope`), + KEY `kind` (`kind`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='References between objects'; + -DROP TABLE IF EXISTS `t_stor_objects_replicas`; CREATE TABLE `t_stor_objects_replicas` ( `object` binary(16) NOT NULL COMMENT 'Object UUID', `device` binary(16) NOT NULL COMMENT 'Device UUID', @@ -112,28 +115,22 @@ CREATE TABLE `t_stor_objects_replicas` ( `ts_updated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Timestamp from when object was last tested/modified on its balanced/consistent status', `ts_locked` timestamp NULL DEFAULT NULL COMMENT 'Timestamp until when object is locked for other operations on device.', `balanced` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Object is present on devices', - `consistent` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Object file size and hash on device matches that which is stored in the database.' + `consistent` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'Object file size and hash on device matches that which is stored in the database.', + UNIQUE KEY `unique_object_device` (`object`,`device`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +CREATE TABLE `t_stor_bucket_dgs` ( + `bucket` binary(16) NOT NULL COMMENT 'Bucket UUID (foreign key)', + `dg` binary(16) NOT NULL COMMENT 'Bucket device group UUID (foreign key)', + UNIQUE KEY `unique_bucket_dg` (`bucket`,`dg`), + KEY `bucket_uuid` (`bucket`), + KEY `dg` (`dg`), + CONSTRAINT `t_stor_bucket_dgs_ibfk_1` FOREIGN KEY (`bucket`) REFERENCES `t_stor_buckets` (`uuid`) ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT `t_stor_bucket_dgs_ibfk_2` FOREIGN KEY (`dg`) REFERENCES `t_stor_dgs` (`uuid`) ON DELETE RESTRICT ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor Buckets are isolated containers for links with assignable storage device groups (dgs)'; + + -DROP TABLE IF EXISTS `t_stor_statuslog`; -CREATE TABLE `t_stor_statuslog` ( - `uuid` binary(16) NOT NULL DEFAULT (uuid_to_bin(uuid(),true)) COMMENT 'Status line UUID', - `uuid_dg` binary(16) NOT NULL COMMENT 'Dg UUID', - `uuid_dev` binary(16) NOT NULL COMMENT 'Device UUID', - `prio` int NOT NULL DEFAULT '1000' COMMENT 'Device priority in dg', - `role` varchar(16) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL DEFAULT 'undefined' COMMENT 'Device role dg', - `data` json NOT NULL COMMENT 'Device status JSON', - `health` varchar(16) GENERATED ALWAYS AS (coalesce(json_unquote(json_extract(`data`,_utf8mb4'$.health')),_utf8mb4'undefined')) VIRTUAL COMMENT 'Device health status', - `ts_created` datetime(1) DEFAULT CURRENT_TIMESTAMP(1) COMMENT 'Timestamp Created with precision of 1 second', - `ts_updated` datetime(1) DEFAULT CURRENT_TIMESTAMP(1) ON UPDATE CURRENT_TIMESTAMP(1) COMMENT 'Timestamp Updated with precision of 1 second', - `boundary` bigint GENERATED ALWAYS AS ((floor((`ts_created` / 86400)) * 86400)) STORED COMMENT 'Boundary timestamp rounded to 24 hours (86400 seconds)', - `data_hash` binary(16) GENERATED ALWAYS AS (unhex(md5(`data`))) STORED COMMENT 'Device status hash', - `uniq_hash` binary(16) GENERATED ALWAYS AS (unhex(md5(concat(`uuid_dg`,`uuid_dev`,`data`,`boundary`)))) STORED COMMENT 'Unique row hash', - PRIMARY KEY (`uuid`), - UNIQUE KEY `uk_uniq_hash` (`uniq_hash`), - KEY `idx_health_prefix` (`health`(2)) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Stor devices/dgs status log. Uniqueness based on Device UUID, partial hash, and creation timestamp (24 hours intervals)'; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; diff --git a/glued/Controllers/ServiceController.php b/glued/Controllers/ServiceController.php index 94590ff..6b0fdf5 100644 --- a/glued/Controllers/ServiceController.php +++ b/glued/Controllers/ServiceController.php @@ -318,6 +318,7 @@ public function get_buckets($bucket = null): array $res = $this->mysqli->execute_query($q, $arg ?? []); $data = $res->fetch_all(MYSQLI_ASSOC); + if ($data == []) { return []; } $unflat = []; @@ -378,11 +379,12 @@ public function get_buckets($bucket = null): array } public function buckets_r1(Request $request, Response $response, array $args = []): Response { $params = $request->getQueryParams(); - $unflat = $this->get_buckets($args['id'] ?? null); + $b = $this->get_buckets($args['id'] ?? null); + if ($b == []) { throw new \Exception("Bucket `{$args['id']}` not found.", 404); } $data = [ 'timestamp' => microtime(), 'status' => 'Buckets r1 OK', - 'data' => $unflat, + 'data' => $b, 'service' => basename(__ROOT__), ]; return $response->withJson($data); @@ -451,15 +453,44 @@ private function get_hardlink($file, $bucket = ''): array public function objects_r1(Request $request, Response $response, array $args = []): Response { - $params = $request->getQueryParams(); - $file = '/var/www/html/data/glued-stor/data/hashes/7/5/1/2/7512b4c69ad0d6c38da365fcb41c0f2b9642c34f2e92183b122c1955e0aeb077fb3caa0678d9597a4b069e178fd990db5048851906fed06fdf82f4eb37a4dded'; - $h = $this->get_hardlink($file); + + if (!array_key_exists('bucket', $args)) { throw new Exception('Bucket not found.', 400); } + $wm = ''; + $link = false; + $pa = [ $args['bucket'] ]; + if (array_key_exists('object', $args)) { + $wm = "AND object = uuid_to_bin(? ,1)"; + $pa[] = $args['object']; + if (array_key_exists('link', $args)) { + $link = $this->generateRetrievalUri($args['object'], $args['bucket']); + } + } + + $q = " + SELECT + -- bin_to_uuid(o.`bucket`,1) AS `bucket`, + -- HEX(o.`hash`) AS `hash`, + bin_to_uuid(o.`object`,1) AS `object`, + f.c_size as size, + f.c_mime as mime, + o.name as name, + f.c_ext as ext, + o.ts_created as created + FROM `t_stor_objects` o + LEFT JOIN t_stor_files f ON f.c_hash = o.hash + WHERE bucket = uuid_to_bin(? ,1) {$wm} + "; + + $handle = $this->mysqli->execute_query($q, $pa); + $r = $handle->fetch_all(MYSQLI_ASSOC); + + if ($link) { $r['link'] = $link; } + $data = [ 'timestamp' => microtime(), 'status' => 'Objects R1 OK', - 'params' => $params, 'service' => basename(__ROOT__), - 'data' => $h + 'data' => $r ]; return $response->withJson($data); } @@ -476,11 +507,13 @@ public function links_r1(Request $request, Response $response, array $args = []) if (!array_key_exists('exp', $args)) { throw new Exception('Expiry not defined.', 400); } $res = $this->decodeRetrievalKey("{$args['token']}/{$args['nonce']}/{$args['exp']}"); if (is_null($res)) { $data['status'] = 'Not found'; return $response->withJson($data)->withStatus(404); } + if ($args['exp'] < time()) { $data['status'] = 'Gone'; return $response->withJson($data)->withStatus(410); } $tree = $this->name_to_path($res['obj']); $file = "/var/www/html/data/glued-stor/data/buckets/{$res['bkt']}/$tree/{$res['obj']}"; - $mime = mime_content_type($file); + if (file_exists($file)) { + $mime = mime_content_type($file); $response = $response ->withHeader('Content-Type', $mime) ->withHeader('Content-Disposition', "attachment;filename=\"{$res['obj']}.{$mime}\"") @@ -504,7 +537,6 @@ function object_meta($file): array { $path = $file->getStream()->getMetadata('uri'); $mime = mime_content_type($path) ?? $file->getClientMediaType() ?? null; $data = [ - 'uuid' => Uuid::uuid4()->toString(), 'name' => $file->getClientFilename(), 'size' => $file->getSize(), 'hash' => [ @@ -567,7 +599,7 @@ public function decodeRetrievalKey(string $token): ?array } - private function write_object($file, $bucket): array { + private function write_object($file, $bucket, $meta): array { // Filter writable devices (for now, only filesystem devices are allowed) foreach ($bucket['devices'] as &$dev) { $dev['path'] = $this->devices($dev['uuid'])['path']; } @@ -594,41 +626,78 @@ private function write_object($file, $bucket): array { // Ensure hashFile is hardlinked as objectFile. If objectFile already exists, do not use the generated objectUUID $objFile = $this->get_hardlink($hashFile, $bucket['uuid'])[0] ?? null; - if (!is_null($objFile)) { $object['uuid'] = basename($objFile); } + if (!is_null($objFile)) { $objUUID = basename($objFile); } else { - $objDir = "{$device['path']}/buckets/{$bucket['uuid']}/{$this->name_to_path($object['uuid'])}"; - $objFile = "{$objDir}/{$object['uuid']}"; + $objUUID = Uuid::uuid4()->toString(); + $objDir = "{$device['path']}/buckets/{$bucket['uuid']}/{$this->name_to_path($objUUID)}"; + $objFile = "{$objDir}/{$objUUID}"; if (!is_dir($objDir)) { mkdir($objDir, 0777, true); } link($hashFile, $objFile); } - $object['link'] = $this->generateRetrievalUri($object['uuid'], $bucket['uuid']); } else { throw new \Exception('No suitable local device found'); } + $q0 = " + INSERT INTO `t_stor_files` (`c_data`) VALUES (?) ON DUPLICATE KEY UPDATE `c_data` = ?; + "; $q1 = " INSERT INTO `t_stor_objects_replicas` (`object`, `device`, `balanced`, `consistent`) VALUES (uuid_to_bin(?, 1), uuid_to_bin(?, 1), ?, ?) ON DUPLICATE KEY UPDATE `balanced` = ?, `consistent` = ?; "; + $q2 = " + INSERT INTO `t_stor_objects_replicas` + (`object`, `device`, `balanced`, `consistent`) VALUES (uuid_to_bin(?, 1), uuid_to_bin(?, 1), ?, ?) + ON DUPLICATE KEY UPDATE `ts_updated` = NOW(); + "; + + $q3 = " + INSERT INTO `t_stor_objects` (`bucket`,`object`,`hash`,`name`) VALUES (uuid_to_bin(?, 1), uuid_to_bin(?, 1), unhex(?), ?) ON DUPLICATE KEY UPDATE `name` = VALUES(`name`); + "; $this->mysqli->begin_transaction(); + $s0 = $this->mysqli->prepare($q0); $s1 = $this->mysqli->prepare($q1); - //$s2 = $this->mysqli->prepare($q2); - //$s3 = $this->mysqli->prepare($q3); - $o = $object['uuid']; + $s2 = $this->mysqli->prepare($q2); + $s3 = $this->mysqli->prepare($q3); + + $v0 = 0; + $v1 = 1; + $fn = $file->getClientFilename(); $d = $device['uuid']; - $v = 1; - $s1->bind_param("ssiiii", $o, $d, $v, $v, $v, $v); + $jo = json_encode($object); + $b = $bucket['uuid']; + $h = $object['hash']['sha3-512']; + + $s0->bind_param("ss", $jo, $jo); + $s0->execute(); + + $s1->bind_param("ssiiii", $objUUID, $d, $v1, $v1, $v1, $v1); $s1->execute(); + + foreach ($bucket['devices'] as $dev) { + if ($dev['uuid'] == $d) continue; + $dd = $dev['uuid']; + $s2->bind_param("ssii", $objUUID, $dd, $v0, $v0); + $s2->execute(); + } + $s3->bind_param("ssss",$b,$objUUID,$h,$fn); + $s3->execute(); + $this->mysqli->commit(); + + $object['uuid'] = $objUUID; + $object['link'] = $this->generateRetrievalUri($objUUID, $bucket['uuid']); + $object['refs'] = $meta; + return $object; } + // TODO t_core_tokens now has only a c_inherit column. we'll need a c_owner column too, because user tokens will use c_inherit, but svc tokens will have c_owner (the one whom is the token management associated to) public function objects_c1(Request $request, Response $response, array $args = []): Response { - // initial sanity checks - if (!array_key_exists('bucket', $args)) { throw new Exception('Bucket not defined.', 500); } + if (!array_key_exists('bucket', $args)) { throw new Exception('Bucket not found.', 400); } if (isset($_SERVER['CONTENT_LENGTH']) && (int) $_SERVER['CONTENT_LENGTH'] > $this->convert_units(ini_get('post_max_size'))) { throw new Exception( 'Upload size in bytes exceeds allowed limit ('.$_SERVER['CONTENT_LENGTH'].' > '.$this->convert_units(ini_get('post_max_size')).').', @@ -637,6 +706,7 @@ public function objects_c1(Request $request, Response $response, array $args = [ $bucket = $this->get_buckets($args['bucket']); + if ($bucket == []) { throw new \Exception("Bucket `{$args['bucket']}` not found.", 404); } $data['bucket'] = $bucket; //return $response->withJson($data); @@ -650,6 +720,7 @@ public function objects_c1(Request $request, Response $response, array $args = [ $headerValueArray = $request->getHeader('Accept'); $headerValueString = $request->getHeaderLine('Accept'); + $meta = json_decode($parsedBody['meta'] ?? '{}', true); if (!empty($files)) { @@ -661,13 +732,9 @@ public function objects_c1(Request $request, Response $response, array $args = [ foreach ($flattened as $file) { if ($file->getError() === UPLOAD_ERR_OK) { - $data['files'][] = $this->write_object($file, $bucket); - - /* - //print_r($data); echo "DIEING"; die(); - //$f['meta'] = json_decode($parsedBody['stor'] ?? [],true)[$file->getClientFilename()] ?? []; + $res = $this->write_object($file, $bucket, $meta[$file->getClientFilename()] ?? []); + $data[] = $res; -*/ } else { // Add error message to response array $data[] = array(