Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Azure short-lived token support #8522

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Azure short-lived token support #8522

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/agent/agent.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,8 @@ class Agent {
params.cloud_info.endpoint_type === 'IBM_COS') {
this.node_type = 'BLOCK_STORE_S3';
this.block_store = new BlockStoreS3(block_store_options);
} else if (params.cloud_info.endpoint_type === 'AZURE') {
} else if (params.cloud_info.endpoint_type === 'AZURE' ||
params.cloud_info.endpoint_type === 'AZURESTS') {
const connection_string = cloud_utils.get_azure_new_connection_string({
endpoint: params.cloud_info.endpoint,
access_key: params.cloud_info.access_keys.access_key,
Expand Down
16 changes: 16 additions & 0 deletions src/api/account_api.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,10 @@ module.exports = {
identity: { $ref: 'common_api#/definitions/access_key' },
secret: { $ref: 'common_api#/definitions/secret_key' },
azure_log_access_keys: { $ref: 'common_api#/definitions/azure_log_access_keys' },
azure_subscription_id: { type: 'string' },
azure_tenant_id: { type: 'string' },
azure_client_id: { type: 'string' },
azure_region: { type: 'string' },
aws_sts_arn: {
type: 'string'
},
Expand Down Expand Up @@ -508,6 +512,18 @@ module.exports = {
identity: { $ref: 'common_api#/definitions/access_key' },
secret: { $ref: 'common_api#/definitions/secret_key' },
azure_log_access_keys: { $ref: 'common_api#/definitions/azure_log_access_keys' },
azure_subscription_id: {
type: 'string'
},
azure_tenant_id: {
type: 'string'
},
azure_client_id: {
type: 'string'
},
azure_region: {
type: 'string'
},
aws_sts_arn: {
type: 'string'
},
Expand Down
2 changes: 1 addition & 1 deletion src/api/common_api.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -751,7 +751,7 @@ module.exports = {

endpoint_type: {
type: 'string',
enum: ['AWSSTS', 'AWS', 'AZURE', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'NET_STORAGE', 'IBM_COS']
enum: ['AWSSTS', 'AWS', 'AZURE', 'AZURESTS', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'NET_STORAGE', 'IBM_COS']
},

block_md: {
Expand Down
13 changes: 12 additions & 1 deletion src/hosted_agents/hosted_agents.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,10 @@ class HostedAgents {
secret_key: pool.cloud_pool_info.access_keys.secret_key
},
aws_sts_arn: pool.cloud_pool_info.aws_sts_arn,
azure_client_id: pool.cloud_pool_info.azure_client_id,
azure_tenant_id: pool.cloud_pool_info.azure_tenant_id,
azure_region: pool.cloud_pool_info.azure_region,
azure_subscription_id: pool.cloud_pool_info.azure_subscription_id,
region: pool.cloud_pool_info.region,
pool_name: pool.name
} : {
Expand All @@ -174,14 +178,21 @@ class HostedAgents {
node_name,
host_id,
storage_path,
token_wrapper,
token_wrapper,
create_node_token_wrapper,
routing_hint: 'LOOPBACK'
};
agent_params[pool_path_property] = pool_path;
agent_params[pool_info_property] = pool_info;
if (pool.cloud_pool_info && pool.cloud_pool_info.storage_limit) agent_params.storage_limit = pool.cloud_pool_info.storage_limit;
if (pool.cloud_pool_info && pool.cloud_pool_info.aws_sts_arn) agent_params.aws_sts_arn = pool.cloud_pool_info.aws_sts_arn;
if (pool.cloud_pool_info) {
const { azure_client_id, azure_tenant_id, azure_region, azure_subscription_id } = pool.cloud_pool_info;
if (azure_client_id) agent_params.azure_client_id = azure_client_id;
if (azure_tenant_id) agent_params.azure_tenant_id = azure_tenant_id;
if (azure_region) agent_params.azure_region = azure_region;
if (azure_subscription_id) agent_params.azure_subscription_id = azure_subscription_id;
}
dbg.log0(`running agent with params ${util.inspect(agent_params)}`);
const agent = new Agent(agent_params);
this._started_agents[node_name] = {
Expand Down
14 changes: 14 additions & 0 deletions src/server/system_services/account_server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -784,6 +784,16 @@ async function add_external_connection(req) {
};
}

if (req.rpc_params.azure_subscription_id &&
req.rpc_params.azure_tenant_id &&
req.rpc_params.azure_client_id &&
req.rpc_params.azure_client_secret) {
info.azure_subscription_id = req.rpc_params.azure_subscription_id;
info.azure_tenant_id = req.rpc_params.azure_tenant_id;
info.azure_client_id = req.rpc_params.azure_client_id;
info.azure_region = req.rpc_params.azure_region;
}

info.cp_code = req.rpc_params.cp_code || undefined;
info.auth_method = req.rpc_params.auth_method || config.DEFAULT_S3_AUTH_METHOD[info.endpoint_type] || undefined;
info = _.omitBy(info, _.isUndefined);
Expand Down Expand Up @@ -958,6 +968,10 @@ async function _check_external_connection_internal(connection) {
case 'AZURE': {
return check_azure_connection(connection);
}
case 'AZURESTS': {
// TODO - Requires example secret to implement
return check_azure_sts_connection(connection);
}
case 'AWSSTS': {
return check_aws_sts_connection(connection);
}
Expand Down
1 change: 1 addition & 0 deletions src/server/system_services/bucket_server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1196,6 +1196,7 @@ async function get_cloud_buckets(req) {
req.rpc_params.connection
);
if (connection.endpoint_type === 'AZURE') {
// TODO - utilize CCO secret for authentication
const blob_svc = azure_storage.BlobServiceClient.fromConnectionString(
cloud_utils.get_azure_new_connection_string(connection));
const used_cloud_buckets = cloud_utils.get_used_cloud_targets(['AZURE'],
Expand Down
5 changes: 5 additions & 0 deletions src/server/system_services/pool_server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,10 @@ async function create_cloud_pool(req) {
account_id: req.account._id
},
aws_sts_arn: connection.aws_sts_arn,
azure_subscription_id: connection.azure_subscription_id,
azure_tenant_id: connection.azure_tenant_id,
azure_client_id: connection.azure_client_id,
azure_region: connection.azure_region,
region: connection.region,
endpoint_type: connection.endpoint_type || 'AWS',
backingstore: req.rpc_params.backingstore,
Expand Down Expand Up @@ -416,6 +420,7 @@ async function create_cloud_pool(req) {
FLASHBLADE: 'BLOCK_STORE_S3',
IBM_COS: 'BLOCK_STORE_S3',
AZURE: 'BLOCK_STORE_AZURE',
AZURESTS: 'BLOCK_STORE_AZURE',
GOOGLE: 'BLOCK_STORE_GOOGLE'
};

Expand Down
6 changes: 5 additions & 1 deletion src/server/system_services/schemas/account_schema.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,10 @@ module.exports = {
access_key: { $ref: 'common_api#/definitions/access_key' },
secret_key: { $ref: 'common_api#/definitions/secret_key' },
azure_log_access_keys: { $ref: 'common_api#/definitions/azure_log_access_keys' },
azure_subscription_id: { type: 'string' },
azure_tenant_id: { type: 'string' },
azure_client_id: { type: 'string' },
azure_region: { type: 'string' },
aws_sts_arn: {
type: 'string'
},
Expand All @@ -82,7 +86,7 @@ module.exports = {
cp_code: { type: 'string' },
endpoint_type: {
type: 'string',
enum: ['AWSSTS', 'AWS', 'AZURE', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'NET_STORAGE', 'IBM_COS']
enum: ['AWSSTS', 'AWS', 'AZURE', 'AZURESTS', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'NET_STORAGE', 'IBM_COS']
},
}
}
Expand Down
6 changes: 5 additions & 1 deletion src/server/system_services/schemas/pool_schema.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,10 @@ module.exports = {
aws_sts_arn: {
type: 'string'
},
azure_subscription_id: { type: 'string' },
azure_tenant_id: { type: 'string' },
azure_client_id: { type: 'string' },
azure_region: { type: 'string' },
backingstore: {
type: 'object',
properties: {
Expand Down Expand Up @@ -138,7 +142,7 @@ module.exports = {
},
endpoint_type: {
type: 'string',
enum: ['AWSSTS', 'AWS', 'AZURE', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'NET_STORAGE', 'IBM_COS']
enum: ['AWSSTS', 'AWS', 'AZURE', 'AZURESTS', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'NET_STORAGE', 'IBM_COS']
},
agent_info: {
type: 'object',
Expand Down
10 changes: 10 additions & 0 deletions src/util/cloud_utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,14 @@ async function generate_aws_sts_creds(params, roleSessionName) {
);
}

async function createAzureSTSClient(params, additionalParams) {
//TODO - Requires example secret to implement
}

async function generate_azure_sts_creds(params, roleSessionName) {
//TODO - Requires example secret to implement
}

function get_signed_url(params, expiry = 604800) {
const s3 = new AWS.S3({
endpoint: params.endpoint,
Expand Down Expand Up @@ -205,3 +213,5 @@ exports.set_noobaa_s3_connection = set_noobaa_s3_connection;
exports.createSTSS3Client = createSTSS3Client;
exports.generate_aws_sts_creds = generate_aws_sts_creds;
exports.generate_access_keys = generate_access_keys;
exports.createSTSS3Client = createSTSS3Client;
exports.createAzureSTSClient = createAzureSTSClient;
Loading