NSFS | NC | IAM Service - Phase 1 (dummy impls)

config.js

@@ -835,6 +835,7 @@ config.NSFS_NC_STORAGE_BACKEND = '';
 config.ENDPOINT_PORT = Number(process.env.ENDPOINT_PORT) || 6001;
 config.ENDPOINT_SSL_PORT = Number(process.env.ENDPOINT_SSL_PORT) || 6443;
 config.ENDPOINT_SSL_STS_PORT = Number(process.env.ENDPOINT_SSL_STS_PORT) || -1;
+config.ENDPOINT_SSL_IAM_PORT = Number(process.env.ENDPOINT_SSL_IAM_PORT) || -1;
 config.ALLOW_HTTP = false;
 // config files should allow access to the owner of the files 
 config.BASE_MODE_CONFIG_FILE = 0o600;

docs/design/iam.md

@@ -0,0 +1,55 @@
+# IAM
+
+## Glossary
+**Access keys** = a pair of access key ID (in short: access key) and secret access key (in short: secret key) 
+**ARN** = Amazon Resource Name 
+**CRUD** = Create, Read, Update, Delete 
+**IAM** = Identity and Access Management 
+**NC** = Non-Containerized 
+**NSFS** = Namespace Store File System 
+
+
+## Goal
+Ability to operate NooBaa accounts for NC NSFS using IAM API ([AWS documentation](https://docs.aws.amazon.com/iam/)). 
+A created user will be able to get access to NooBaa resources (buckets, objects).
+
+## Background
+- Currently, we create NC NSFS accounts using the Manage NSFS, which is a CLI command with root (privileged) permissions:
+ ```bash
+ sudo node src/cmd/manage_nsfs account add [flags]
+ ```
+- The NS NSFS account is saved as a JSON file with root permissions (under the default path: `/etc/noobaa.conf.d/accounts/<name>.json`).
+- The structure of a valid account is determined by schema and validated using avj. 
+There are a couple of required properties specific to NSFS: `nsfs_account_config` that include a UID and GID or a Distinguished Name.
+- When an account is created the json reply contains all the details of the created account (as they are stored in the JSON file).
+
+## Problem
+As mentioned, for NooBaa NC NSFS deployments, the only way to create and update accounts is via the CLI. 
+For certain deployments exposing the CLI is not a viable option (for security reasons, some organizations disable the SSH to a machine with root permissions).
+
+## Scenarios
+### In Scope
+Support IAM API: 
+- CreateUser, GetUser, UpdateUser, DeleteUser, ListUsers. 
+- CreateAccessKey, GetAccessKeyLastUsed, UpdateAccessKey, DeleteAccessKey, ListAccessKeys.
+### Out of Scope
+At this point we will not support additional IAM resources (group, policy, role, etc).
+
+## Architecture
+![IAM FLOW](./images/IamCreateUserSd.png)
+
+- The boilerplate code is based on STS and S3 services 
+- IAM service will be supported in NSFS service (which requires the endpoint)
+- In the endpoint we created the `https_server_iam`
+- The server would listen to a new port `https_port_iam`
+ - It will be a separate port 
+ - During development phase will default to -1 to avoid listening to the port
+- To create the server we created the `endpoint_request_handler_iam`.
+ - The `iam_rest` that either `handle_request` or `handle_error`
+ - The `IamError` class.
+ - The the ops directory and each supported action will be a file with name `iam_<action>`
+- We created the `AccountSDK` class and the `AccountSpace` interface:
+ - The `AccountSpace` interface is defined in `nb.d.ts`
+ - The initial (current) implementation is only `AccountSpaceFS`
+ - `AccountSpaceFS` will contain all our implementations related to users and access keys - like we have for other resources: `NamespaceFS` for objects, `BucketSpaceFS` for buckets, etc
+

docs/design/uml/IamCreateUserSd.puml

@@ -0,0 +1,23 @@
+@startuml
+/'To generate png file, use https://www.plantuml.com/plantuml '/
+
+title Create User Using IAM
+
+actor "client (root user)" as C order 10
+participant "IAM Server (endpoint)" as IS order 20
+participant "endpoint_request_handler" as ERH order 30
+participant "AccountSpace" as ASG order 40
+participant "AccountSpaceFS" as ASF order 50
+participant "FileSystem" as FS order 60
+
+C -> IS: CreateUser request
+IS -> ERH: handle_request
+ERH -> ASG: create_user
+ASG -> ASF: create_user
+ASF -> FS: create_config_file
+note right: usable after CreateAccessKey
+ASF -> ERH: creation_details
+ERH -> IS: send_reply
+IS -> C: CreateUser reply
+
+@enduml

src/cmd/nsfs.js

@@ -41,6 +41,8 @@ const path = require('path');
 const json_utils = require('../util/json_utils');
 //const { RPC_BUFFERS } = require('../rpc');
 const pkg = require('../../package.json');
+const AccountSDK = require('../sdk/account_sdk');
+const AccountSpaceFS = require('../sdk/accountspace_fs');
 const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;
 
 const HELP = `
@@ -70,6 +72,7 @@ Options:
  --http_port <port> (default 6001) Set the S3 endpoint listening HTTP port to serve.
  --https_port <port> (default 6443) Set the S3 endpoint listening HTTPS port to serve.
  --https_port_sts <port> (default -1) Set the S3 endpoint listening HTTPS port for STS.
+ --https_port_iam <port> (default -1) Set the endpoint listening HTTPS port for IAM.
  --metrics_port <port> (default -1) Set the metrics listening port for prometheus.
  --forks <n> (default none) Forks spread incoming requests (config.ENDPOINT_FORKS used if flag is not provided).
  --debug <level> (default 0) Increase debug level.
@@ -207,6 +210,33 @@ class NsfsObjectSDK extends ObjectSDK {
  }
 }
 
+// NsfsAccountSDK was based on NsfsObjectSDK
+// simple flow was not implemented
+class NsfsAccountSDK extends AccountSDK {
+ constructor(fs_root, fs_config, account, config_root) {
+ let bucketspace;
+ let accountspace;
+ if (config_root) {
+ bucketspace = new BucketSpaceFS({ config_root });
+ accountspace = new AccountSpaceFS({ config_root });
+ } else {
+ bucketspace = new BucketSpaceSimpleFS({ fs_root });
+ accountspace = new AccountSpaceFS({ fs_root });
+ }
+ super({
+ rpc_client: null,
+ internal_rpc_client: null,
+ bucketspace: bucketspace,
+ accountspace: accountspace,
+ });
+ this.nsfs_config_root = nsfs_config_root;
+ this.nsfs_fs_root = fs_root;
+ this.nsfs_fs_config = fs_config;
+ this.nsfs_account = account;
+ this.nsfs_namespaces = {};
+ }
+}
+
 async function init_nsfs_system(config_root) {
  const system_data_path = path.join(config_root, 'system.json');
  const system_data = new json_utils.JsonFileWrapper(system_data_path);
@@ -262,6 +292,7 @@ async function main(argv = minimist(process.argv.slice(2))) {
  const http_port = Number(argv.http_port) || config.ENDPOINT_PORT;
  const https_port = Number(argv.https_port) || config.ENDPOINT_SSL_PORT;
  const https_port_sts = Number(argv.https_port_sts) || config.ENDPOINT_SSL_STS_PORT;
+ const https_port_iam = Number(argv.https_port_iam) || config.ENDPOINT_SSL_IAM_PORT;
  const metrics_port = Number(argv.metrics_port) || config.EP_METRICS_SERVER_PORT;
  const forks = Number(argv.forks) || config.ENDPOINT_FORKS;
  if (forks > 0) process.env.ENDPOINT_FORKS = forks.toString(); // used for argv.forks to take effect
@@ -307,6 +338,7 @@ async function main(argv = minimist(process.argv.slice(2))) {
  http_port,
  https_port,
  https_port_sts,
+ https_port_iam,
  metrics_port,
  backend,
  forks,
@@ -324,17 +356,22 @@ async function main(argv = minimist(process.argv.slice(2))) {
  http_port,
  https_port,
  https_port_sts,
+ https_port_iam,
  metrics_port,
  forks,
  nsfs_config_root,
  init_request_sdk: (req, res) => {
  req.object_sdk = new NsfsObjectSDK(fs_root, fs_config, account, versioning, nsfs_config_root);
+ req.account_sdk = new NsfsAccountSDK(fs_root, fs_config, account, nsfs_config_root);
  }
  });
  if (config.ALLOW_HTTP) {
  console.log('nsfs: listening on', util.inspect(`http://localhost:${http_port}`));
  }
  console.log('nsfs: listening on', util.inspect(`https://localhost:${https_port}`));
+ if (https_port_iam > 0) {
+ console.log('nsfs: IAM listening on', util.inspect(`https://localhost:${https_port_iam}`));
+ }
  } catch (err) {
  console.error('nsfs: exit on error', err.stack || err);
  //noobaa crashed

src/endpoint/endpoint.js

@@ -20,6 +20,7 @@ const config = require('../../config');
 const s3_rest = require('./s3/s3_rest');
 const blob_rest = require('./blob/blob_rest');
 const sts_rest = require('./sts/sts_rest');
+const iam_rest = require('./iam/iam_rest');
 const lambda_rest = require('./lambda/lambda_rest');
 const endpoint_utils = require('./endpoint_utils');
 const FuncSDK = require('../sdk/func_sdk');
@@ -77,6 +78,7 @@ dbg.log0('endpoint: replacing old umask: ', old_umask.toString(8), 'with new uma
  * http_port?: number;
  * https_port?: number;
  * https_port_sts?: number;
+ * https_port_iam?: number;
  * metrics_port?: number;
  * nsfs_config_root?: string;
  * init_request_sdk?: EndpointHandler;
@@ -98,6 +100,7 @@ async function main(options = {}) {
  const http_port = options.http_port || Number(process.env.ENDPOINT_PORT) || 6001;
  const https_port = options.https_port || Number(process.env.ENDPOINT_SSL_PORT) || 6443;
  const https_port_sts = options.https_port_sts || Number(process.env.ENDPOINT_SSL_PORT_STS) || 7443;
+ const https_port_iam = options.https_port_iam || Number(process.env.ENDPOINT_SSL_PORT_IAM) || 7444;
  const endpoint_group_id = process.env.ENDPOINT_GROUP_ID || 'default-endpoint-group';
 
  const virtual_hosts = Object.freeze(
@@ -183,6 +186,13 @@ async function main(options = {}) {
  await listen_http(https_port_sts, https_server_sts);
  dbg.log0('Started STS HTTPS successfully');
  }
+ if (https_port_iam > 0) {
+ dbg.log0('Starting IAM HTTPS', https_port_iam);
+ const endpoint_request_handler_iam = create_endpoint_handler_iam(init_request_sdk);
+ const https_server_iam = https.createServer(ssl_options, endpoint_request_handler_iam);
+ await listen_http(https_port_iam, https_server_iam);
+ dbg.log0('Started IAM HTTPS successfully');
+ }
  if (metrics_port > 0 && cluster.isPrimary) {
  dbg.log0('Starting metrics server', metrics_port);
  await prom_reporting.start_server(metrics_port, false);
@@ -254,6 +264,16 @@ function create_endpoint_handler(init_request_sdk, virtual_hosts, sts) {
  return sts ? endpoint_sts_request_handler : endpoint_request_handler;
 }
 
+function create_endpoint_handler_iam(init_request_sdk) {
+ /** @type {EndpointHandler} */
+ const endpoint_iam_request_handler = (req, res) => {
+ endpoint_utils.prepare_rest_request(req);
+ init_request_sdk(req, res);
+ return iam_rest(req, res);
+ };
+ return endpoint_iam_request_handler;
+}
+
 function endpoint_fork_id_handler(req, res) {
  let reply = {};
  if (cluster.isWorker) {
@@ -480,6 +500,7 @@ function setup_http_server(server) {
 
 exports.main = main;
 exports.create_endpoint_handler = create_endpoint_handler;
+exports.create_endpoint_handler_iam = create_endpoint_handler_iam;
 exports.create_init_request_sdk = create_init_request_sdk;
 
 if (require.main === module) main();