Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add some development options to SQL metastore #2221

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add some development options to SQL metastore #2221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions src/blob/SqlBlobConfiguration.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ export default class SqlBlobConfiguration extends ConfigurationBase {
key: string = "",
pwd: string = "",
oauth?: string,
disableProductStyleUrl: boolean = false
disableProductStyleUrl: boolean = false,
clearDB: boolean = false,
softDelete: boolean = true
) {
super(
host,
Expand All @@ -50,7 +52,9 @@ export default class SqlBlobConfiguration extends ConfigurationBase {
key,
pwd,
oauth,
disableProductStyleUrl
disableProductStyleUrl,
clearDB,
softDelete
);
}
}
7 changes: 5 additions & 2 deletions src/blob/SqlBlobServer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,17 @@ export default class SqlBlobServer extends ServerBase {

const metadataStore: IBlobMetadataStore = new SqlBlobMetadataStore(
configuration.sqlURL,
configuration.sequelizeOptions
configuration.sequelizeOptions,
configuration.clearDB,
configuration.softDelete
);

const extentMetadataStore: IExtentMetadataStore = new SqlExtentMetadataStore(
// Currently, extent metadata and blob metadata share same database
// But they can use separate databases per future requirements
configuration.sqlURL,
configuration.sequelizeOptions
configuration.sequelizeOptions,
configuration.clearDB
);

const extentStore: IExtentStore = new FSExtentStore(
Expand Down
217 changes: 91 additions & 126 deletions src/blob/persistence/SqlBlobMetadataStore.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,9 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore {
*/
public constructor(
connectionURI: string,
sequelizeOptions?: SequelizeOptions
sequelizeOptions?: SequelizeOptions,
private readonly clearDB: boolean = false,
private readonly softDelete: boolean = true
) {
// Enable encrypt connection for SQL Server
if (connectionURI.startsWith("mssql") && sequelizeOptions) {
Expand Down Expand Up @@ -365,8 +367,12 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore {
}
);

// TODO: sync() is only for development purpose, use migration for production
await this.sequelize.sync();
if (this.clearDB) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Blob content is on local disk, so clear DB will make data not aligned.

await this.sequelize.sync({ force: true });
} else {
// TODO: sync() is only for development purpose, use migration for production
await this.sequelize.sync();
}

this.initialized = true;
}
Expand Down Expand Up @@ -642,33 +648,8 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore {
transaction: t
});

// TODO: GC blobs under deleting status
await BlobsModel.update(
{
deleting: literal("deleting + 1")
},
{
where: {
accountName: account,
containerName: container
},
transaction: t
}
);

// TODO: GC blocks under deleting status
await BlocksModel.update(
{
deleting: literal("deleting + 1")
},
{
where: {
accountName: account,
containerName: container
},
transaction: t
}
);
await this.destroyBlob(account, container, undefined, undefined, t);
await this.destroyBlock(account, container, undefined, undefined, t);
/* Transaction ends */
});
}
Expand Down Expand Up @@ -1439,7 +1420,7 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore {
});
}

public getBlockList(
public async getBlockList(
context: Context,
account: string,
container: string,
Expand Down Expand Up @@ -1667,18 +1648,12 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore {
transaction: t
});

await BlocksModel.update(
{
deleting: literal("deleting + 1")
},
{
where: {
accountName: blob.accountName,
containerName: blob.containerName,
blobName: blob.name
},
transaction: t
}
await this.destroyBlock(
blob.accountName,
blob.containerName,
blob.name,
undefined,
t
);
});
}
Expand Down Expand Up @@ -1891,107 +1866,31 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore {
if (count > 1) {
throw StorageErrorFactory.getSnapshotsPresent(context.contextId!);
} else {
await BlobsModel.update(
{
deleting: literal("deleting + 1")
},
{
where: {
accountName: account,
containerName: container,
blobName: blob
},
transaction: t
}
);

await BlocksModel.update(
{
deleting: literal("deleting + 1")
},
{
where: {
accountName: account,
containerName: container,
blobName: blob
},
transaction: t
}
);
await this.destroyBlob(account, container, blob, undefined, t);
await this.destroyBlock(account, container, blob, undefined, t);
}
}

// Scenario: Delete one snapshot only
if (!againstBaseBlob) {
await BlobsModel.update(
{
deleting: literal("deleting + 1")
},
{
where: {
accountName: account,
containerName: container,
blobName: blob,
snapshot: blobModel.snapshot
},
transaction: t
}
);
await this.destroyBlob(account, container, blob, blobModel.snapshot, t);
}

// Scenario: Delete base blob and snapshots
if (
againstBaseBlob &&
options.deleteSnapshots === Models.DeleteSnapshotsOptionType.Include
) {
await BlobsModel.update(
{
deleting: literal("deleting + 1")
},
{
where: {
accountName: account,
containerName: container,
blobName: blob
},
transaction: t
}
);

await BlocksModel.update(
{
deleting: literal("deleting + 1")
},
{
where: {
accountName: account,
containerName: container,
blobName: blob
},
transaction: t
}
);
await this.destroyBlob(account, container, blob, undefined, t);
await this.destroyBlock(account, container, blob, undefined, t);
}

// Scenario: Delete all snapshots only
if (
againstBaseBlob &&
options.deleteSnapshots === Models.DeleteSnapshotsOptionType.Only
) {
await BlobsModel.update(
{
deleting: literal("deleting + 1")
},
{
where: {
accountName: account,
containerName: container,
blobName: blob,
snapshot: { [Op.gt]: "" }
},
transaction: t
}
);
await this.destroyBlob(account, container, blob, { [Op.gt]: "" }, t);
}
});
}
Expand Down Expand Up @@ -2072,7 +1971,7 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore {
});
}

public setBlobMetadata(
public async setBlobMetadata(
context: Context,
account: string,
container: string,
Expand Down Expand Up @@ -3447,4 +3346,70 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore {
}
return undefined;
}

private async destroyBlob(
account: string,
container: string,
blob: string | undefined,
snapshot: any,
t: Transaction
): Promise<void> {
const where: any = {
accountName: account,
containerName: container
};

if (blob !== undefined) where.blobName = blob;
if (snapshot !== undefined) where.snapshot = snapshot;

if (this.softDelete) {
await BlobsModel.update(
{
deleting: literal("deleting + 1")
},
{
where,
transaction: t
}
);
} else {
await BlobsModel.destroy({
where,
transaction: t
});
}
}

private async destroyBlock(
account: string,
container: string,
blob: string | undefined,
snapshot: any,
t: Transaction
): Promise<void> {
const where: any = {
accountName: account,
containerName: container
};

if (blob !== undefined) where.blobName = blob;
if (snapshot !== undefined) where.snapshot = snapshot;

if (this.softDelete) {
await BlocksModel.update(
{
deleting: literal("deleting + 1")
},
{
where,
transaction: t
}
);
} else {
await BlocksModel.destroy({
where,
transaction: t
});
}
}
}
2 changes: 2 additions & 0 deletions src/common/ConfigurationBase.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ export default abstract class ConfigurationBase {
public readonly pwd: string = "",
public readonly oauth?: string,
public readonly disableProductStyleUrl: boolean = false,
public readonly clearDB: boolean = false,
public readonly softDelete: boolean = true
) {}

public hasCert() {
Expand Down
11 changes: 8 additions & 3 deletions src/common/persistence/SqlExtentMetadataStore.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ export default class SqlExtentMetadataStore implements IExtentMetadataStore {
*/
public constructor(
connectionURI: string,
sequelizeOptions?: SequelizeOptions
sequelizeOptions?: SequelizeOptions,
private readonly clearDB: boolean = false,
) {
// Enable encrypt connection for SQL Server
if (connectionURI.startsWith("mssql") && sequelizeOptions) {
Expand Down Expand Up @@ -74,8 +75,12 @@ export default class SqlExtentMetadataStore implements IExtentMetadataStore {
{ sequelize: this.sequelize, modelName: "Extents", timestamps: false }
);

// TODO: Remove this part which only for test.
await this.sequelize.sync();
if (this.clearDB) {
await this.sequelize.sync({ force: true });
} else {
// TODO: Remove this part which only for test.
await this.sequelize.sync();
}

this.initialized = true;
}
Expand Down
6 changes: 4 additions & 2 deletions tests/BlobTestServerFactory.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ export default class BlobTestServerFactory {
cert,
key,
undefined,
oauth
oauth,
true
);

return new SqlBlobServer(config);
Expand All @@ -65,7 +66,8 @@ export default class BlobTestServerFactory {
cert,
key,
undefined,
oauth
oauth,
true
);
return new BlobServer(config);
}
Expand Down
Loading