Skip to content

Commit

Permalink
NSFS | NC | IAM service (boilerplate only)
Browse files Browse the repository at this point in the history
Signed-off-by: shirady <[email protected]>
  • Loading branch information
shirady committed May 5, 2024
1 parent ff0007a commit 635e0f6
Show file tree
Hide file tree
Showing 21 changed files with 1,273 additions and 0 deletions.
1 change: 1 addition & 0 deletions config.js
Original file line number Diff line number Diff line change
Expand Up @@ -832,6 +832,7 @@ config.NSFS_NC_STORAGE_BACKEND = '';
config.ENDPOINT_PORT = Number(process.env.ENDPOINT_PORT) || 6001;
config.ENDPOINT_SSL_PORT = Number(process.env.ENDPOINT_SSL_PORT) || 6443;
config.ENDPOINT_SSL_STS_PORT = Number(process.env.ENDPOINT_SSL_STS_PORT) || -1;
config.ENDPOINT_SSL_IAM_PORT = Number(process.env.ENDPOINT_SSL_IAM_PORT) || -1;
config.ALLOW_HTTP = false;
// config files should allow access to the owner of the files
config.BASE_MODE_CONFIG_FILE = 0o600;
Expand Down
32 changes: 32 additions & 0 deletions src/cmd/nsfs.js
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ const path = require('path');
const json_utils = require('../util/json_utils');
//const { RPC_BUFFERS } = require('../rpc');
const pkg = require('../../package.json');
const AccountSDK = require('../sdk/account_sdk');
const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;

const HELP = `
Expand Down Expand Up @@ -70,6 +71,7 @@ Options:
--http_port <port> (default 6001) Set the S3 endpoint listening HTTP port to serve.
--https_port <port> (default 6443) Set the S3 endpoint listening HTTPS port to serve.
--https_port_sts <port> (default -1) Set the S3 endpoint listening HTTPS port for STS.
--https_port_iam <port> (default -1) Set the endpoint listening HTTPS port for IAM.
--metrics_port <port> (default -1) Set the metrics listening port for prometheus.
--forks <n> (default none) Forks spread incoming requests (config.ENDPOINT_FORKS used if flag is not provided).
--debug <level> (default 0) Increase debug level.
Expand Down Expand Up @@ -207,6 +209,29 @@ class NsfsObjectSDK extends ObjectSDK {
}
}

// NsfsAccountSDK was based on NsfsObjectSDK
// simple flow was not implemented
class NsfsAccountSDK extends AccountSDK {
constructor(fs_root, fs_config, account, config_root) {
let bucketspace;
if (config_root) {
bucketspace = new BucketSpaceFS({ config_root });
} else {
bucketspace = new BucketSpaceSimpleFS({ fs_root });
}
super({
rpc_client: null,
internal_rpc_client: null,
bucketspace: bucketspace,
});
this.nsfs_config_root = nsfs_config_root;
this.nsfs_fs_root = fs_root;
this.nsfs_fs_config = fs_config;
this.nsfs_account = account;
this.nsfs_namespaces = {};
}
}

async function init_nsfs_system(config_root) {
const system_data_path = path.join(config_root, 'system.json');
const system_data = new json_utils.JsonFileWrapper(system_data_path);
Expand Down Expand Up @@ -262,6 +287,7 @@ async function main(argv = minimist(process.argv.slice(2))) {
const http_port = Number(argv.http_port) || config.ENDPOINT_PORT;
const https_port = Number(argv.https_port) || config.ENDPOINT_SSL_PORT;
const https_port_sts = Number(argv.https_port_sts) || config.ENDPOINT_SSL_STS_PORT;
const https_port_iam = Number(argv.https_port_iam) || config.ENDPOINT_SSL_IAM_PORT;
const metrics_port = Number(argv.metrics_port) || config.EP_METRICS_SERVER_PORT;
const forks = Number(argv.forks) || config.ENDPOINT_FORKS;
if (forks > 0) process.env.ENDPOINT_FORKS = forks.toString(); // used for argv.forks to take effect
Expand Down Expand Up @@ -307,6 +333,7 @@ async function main(argv = minimist(process.argv.slice(2))) {
http_port,
https_port,
https_port_sts,
https_port_iam,
metrics_port,
backend,
forks,
Expand All @@ -324,17 +351,22 @@ async function main(argv = minimist(process.argv.slice(2))) {
http_port,
https_port,
https_port_sts,
https_port_iam,
metrics_port,
forks,
nsfs_config_root,
init_request_sdk: (req, res) => {
req.object_sdk = new NsfsObjectSDK(fs_root, fs_config, account, versioning, nsfs_config_root);
req.account_sdk = new NsfsAccountSDK(fs_root, fs_config, account, nsfs_config_root);
}
});
if (config.ALLOW_HTTP) {
console.log('nsfs: listening on', util.inspect(`http://localhost:${http_port}`));
}
console.log('nsfs: listening on', util.inspect(`https://localhost:${https_port}`));
if (https_port_iam > 0) {
console.log('nsfs: IAM listening on', util.inspect(`https://localhost:${https_port_iam}`));
}
} catch (err) {
console.error('nsfs: exit on error', err.stack || err);
//noobaa crashed
Expand Down
25 changes: 25 additions & 0 deletions src/endpoint/endpoint.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,12 @@ const config = require('../../config');
const s3_rest = require('./s3/s3_rest');
const blob_rest = require('./blob/blob_rest');
const sts_rest = require('./sts/sts_rest');
const iam_rest = require('./iam/iam_rest');
const lambda_rest = require('./lambda/lambda_rest');
const endpoint_utils = require('./endpoint_utils');
const FuncSDK = require('../sdk/func_sdk');
const StsSDK = require('../sdk/sts_sdk');
const AccountSDK = require('../sdk/account_sdk');
const ObjectIO = require('../sdk/object_io');
const ObjectSDK = require('../sdk/object_sdk');
const xml_utils = require('../util/xml_utils');
Expand Down Expand Up @@ -61,6 +63,7 @@ dbg.log0('endpoint: replacing old umask: ', old_umask.toString(8), 'with new uma
* object_sdk?: ObjectSDK;
* func_sdk?: FuncSDK;
* sts_sdk?: StsSDK;
* account_sdk?: AccountSDK;
* virtual_hosts?: readonly string[];
* }} EndpointRequest
*/
Expand All @@ -77,6 +80,7 @@ dbg.log0('endpoint: replacing old umask: ', old_umask.toString(8), 'with new uma
* http_port?: number;
* https_port?: number;
* https_port_sts?: number;
* https_port_iam?: number;
* metrics_port?: number;
* nsfs_config_root?: string;
* init_request_sdk?: EndpointHandler;
Expand All @@ -98,6 +102,7 @@ async function main(options = {}) {
const http_port = options.http_port || Number(process.env.ENDPOINT_PORT) || 6001;
const https_port = options.https_port || Number(process.env.ENDPOINT_SSL_PORT) || 6443;
const https_port_sts = options.https_port_sts || Number(process.env.ENDPOINT_SSL_PORT_STS) || 7443;
const https_port_iam = options.https_port_iam || Number(process.env.ENDPOINT_SSL_PORT_IAM) || 7444;
const endpoint_group_id = process.env.ENDPOINT_GROUP_ID || 'default-endpoint-group';

const virtual_hosts = Object.freeze(
Expand Down Expand Up @@ -152,11 +157,13 @@ async function main(options = {}) {

const endpoint_request_handler = create_endpoint_handler(init_request_sdk, virtual_hosts);
const endpoint_request_handler_sts = create_endpoint_handler(init_request_sdk, virtual_hosts, true);
const endpoint_request_handler_iam = create_endpoint_handler_iam(init_request_sdk);

const ssl_cert_info = await ssl_utils.get_ssl_cert_info('S3', options.nsfs_config_root);
const ssl_options = { ...ssl_cert_info.cert, honorCipherOrder: true };
const https_server = https.createServer(ssl_options, endpoint_request_handler);
const https_server_sts = https.createServer(ssl_options, endpoint_request_handler_sts);
const https_server_iam = https.createServer(ssl_options, endpoint_request_handler_iam);
ssl_cert_info.on('update', updated_ssl_cert_info => {
dbg.log0("Setting updated S3 ssl certs for endpoint.");
const updated_ssl_options = { ...updated_ssl_cert_info.cert, honorCipherOrder: true };
Expand All @@ -183,6 +190,11 @@ async function main(options = {}) {
await listen_http(https_port_sts, https_server_sts);
dbg.log0('Started STS HTTPS successfully');
}
if (https_port_iam > 0) {
dbg.log0('Starting IAM HTTPS', https_port_iam);
await listen_http(https_port_iam, https_server_iam);
dbg.log0('Started IAM HTTPS successfully');
}
if (metrics_port > 0 && cluster.isPrimary) {
dbg.log0('Starting metrics server', metrics_port);
await prom_reporting.start_server(metrics_port, false);
Expand Down Expand Up @@ -254,6 +266,16 @@ function create_endpoint_handler(init_request_sdk, virtual_hosts, sts) {
return sts ? endpoint_sts_request_handler : endpoint_request_handler;
}

function create_endpoint_handler_iam(init_request_sdk) {
/** @type {EndpointHandler} */
const endpoint_iam_request_handler = (req, res) => {
endpoint_utils.prepare_rest_request(req);
init_request_sdk(req, res);
return iam_rest(req, res);
};
return endpoint_iam_request_handler;
}

function endpoint_fork_id_handler(req, res) {
let reply = {};
if (cluster.isWorker) {
Expand Down Expand Up @@ -289,6 +311,8 @@ function create_init_request_sdk(rpc, internal_rpc_client, object_io) {
const rpc_client = rpc.new_client();
req.func_sdk = new FuncSDK(rpc_client);
req.sts_sdk = new StsSDK(rpc_client, internal_rpc_client);
// this flow was not checked (assumed that init_request_sdk was passed) in account_sdk
req.account_sdk = new AccountSDK(rpc_client, internal_rpc_client);
req.object_sdk = new ObjectSDK({
rpc_client,
internal_rpc_client,
Expand Down Expand Up @@ -480,6 +504,7 @@ function setup_http_server(server) {

exports.main = main;
exports.create_endpoint_handler = create_endpoint_handler;
exports.create_endpoint_handler_iam = create_endpoint_handler_iam;
exports.create_init_request_sdk = create_init_request_sdk;

if (require.main === module) main();
148 changes: 148 additions & 0 deletions src/endpoint/iam/iam_errors.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
/* Copyright (C) 2024 NooBaa */
'use strict';
const xml_utils = require('../../util/xml_utils');

// https://docs.aws.amazon.com/IAM/latest/APIReference/CommonErrors.html
/**
* @typedef {{
* code?: string,
* message: string,
* http_code: number,
* detail?: string
* }} IamErrorSpec
*/

class IamError extends Error {

/**
* @param {IamErrorSpec} error_spec
*/
constructor({ code, message, http_code, detail }) {
super(message); // sets this.message
this.code = code;
this.http_code = http_code;
this.detail = detail;
}

reply(resource, request_id) {
const xml = {
Error: {
Code: this.code,
Message: this.message,
Resource: resource || '',
RequestId: request_id || '',
Detail: this.detail,
}
};
return xml_utils.encode_xml(xml);
}

}

IamError.AccessDeniedException = Object.freeze({
code: 'AccessDeniedException',
message: 'You do not have sufficient access to perform this action.',
http_code: 400,
});
IamError.IncompleteSignature = Object.freeze({
code: 'IncompleteSignature',
message: 'The request signature does not conform to AWS standards.',
http_code: 400,
});
IamError.InternalFailure = Object.freeze({
code: 'InternalFailure',
message: 'The request processing has failed because of an unknown error, exception or failure.',
http_code: 500,
});
IamError.InvalidAction = Object.freeze({
code: 'InvalidAction',
message: 'The action or operation requested is invalid. Verify that the action is typed correctly.',
http_code: 400,
});
IamError.InvalidClientTokenId = Object.freeze({
code: 'InvalidClientTokenId',
message: 'The X.509 certificate or AWS access key ID provided does not exist in our records.',
http_code: 403,
});
IamError.NotAuthorized = Object.freeze({
code: 'NotAuthorized',
message: 'You do not have permission to perform this action.',
http_code: 400,
});
IamError.OptInRequired = Object.freeze({
code: 'OptInRequired',
message: 'The AWS access key ID needs a subscription for the service.',
http_code: 403,
});
IamError.RequestExpired = Object.freeze({
code: 'RequestExpired',
message: 'The request reached the service more than 15 minutes after the date stamp on the request or more than 15 minutes after the request expiration date (such as for pre-signed URLs), or the date stamp on the request is more than 15 minutes in the future.',
http_code: 400,
});
IamError.ServiceUnavailable = Object.freeze({
code: 'ServiceUnavailable',
message: 'The request has failed due to a temporary failure of the server.',
http_code: 503,
});
IamError.ThrottlingException = Object.freeze({
code: 'ThrottlingException',
message: 'The request was denied due to request throttling.',
http_code: 400,
});
IamError.ValidationError = Object.freeze({
code: 'ValidationError',
message: 'The input fails to satisfy the constraints specified by an AWS service.',
http_code: 400,
});
// internal error (not appears in the IAM error list)
IamError.NotImplemented = Object.freeze({
code: 'NotImplemented',
message: 'A header you provided implies functionality that is not implemented.',
http_code: 501,
});

// These errors were copied from STS errors
// TODO - can be deleted after verifying we will not use them
IamError.InvalidParameterCombination = Object.freeze({
code: 'InvalidParameterCombination',
message: 'Parameters that must not be used together were used together.',
http_code: 400,
});
IamError.InvalidParameterValue = Object.freeze({
code: 'InvalidParameterValue',
message: 'An invalid or out-of-range value was supplied for the input parameter.',
http_code: 400,
});
IamError.InvalidQueryParameter = Object.freeze({
code: 'InvalidQueryParameter',
message: 'The AWS query string is malformed or does not adhere to AWS standards.',
http_code: 400,
});
IamError.MalformedQueryString = Object.freeze({
code: 'MalformedQueryString',
message: 'The query string contains a syntax error.',
http_code: 404,
});
IamError.MissingAction = Object.freeze({
code: 'MissingAction',
message: 'The request is missing an action or a required parameter.',
http_code: 400,
});
IamError.MissingAuthenticationToken = Object.freeze({
code: 'MissingAuthenticationToken',
message: 'The request must contain either a valid (registered) AWS access key ID or X.509 certificate.',
http_code: 403,
});
IamError.MissingParameter = Object.freeze({
code: 'MissingParameter',
message: 'A required parameter for the specified action is not supplied.',
http_code: 400,
});
IamError.ExpiredToken = Object.freeze({
code: 'ExpiredToken',
message: 'The security token included in the request is expired',
http_code: 400,
});

// EXPORTS
exports.IamError = IamError;

0 comments on commit 635e0f6

Please sign in to comment.