Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added totalBytesSkipped and using this in percentage complete calculation #2856

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+15 −2
Open

Added totalBytesSkipped and using this in percentage complete calculation #2856

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cmd/copy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1859,6 +1859,7 @@ Number of Folder Transfers Failed: %v
Number of File Transfers Skipped: %v
Number of Folder Transfers Skipped: %v
Total Number of Bytes Transferred: %v
Total Number of Bytes Skipped: %v
Final Job Status: %v%s%s
`,
summary.JobID.String(),
Expand All @@ -1874,6 +1875,7 @@ Final Job Status: %v%s%s
summary.TransfersSkipped-summary.FoldersSkipped,
summary.FoldersSkipped,
summary.TotalBytesTransferred,
summary.TotalBytesSkipped,
summary.JobStatus,
screenStats,
formatPerfAdvice(summary.PerformanceAdvice))
Expand Down
2 changes: 2 additions & 0 deletions cmd/jobsResume.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ Number of Folder Transfers Failed: %v
Number of File Transfers Skipped: %v
Number of Folder Transfers Skipped: %v
Total Number of Bytes Transferred: %v
Total Number of Bytes Skipped: %v
Final Job Status: %v
`,
summary.JobID.String(),
Expand All @@ -184,6 +185,7 @@ Final Job Status: %v
summary.TransfersSkipped-summary.FoldersSkipped,
summary.FoldersSkipped,
summary.TotalBytesTransferred,
summary.TotalBytesSkipped,
summary.JobStatus)
}
}, exitCode)
Expand Down
2 changes: 2 additions & 0 deletions cmd/jobsShow.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@ Number of Folder Transfers Failed: %v
Number of File Transfers Skipped: %v
Number of Folder Transfers Skipped: %v
Total Number of Bytes Transferred: %v
Total Number of Bytes Skipped: %v
Percent Complete (approx): %.1f
Final Job Status: %v
`,
Expand All @@ -174,6 +175,7 @@ Final Job Status: %v
summary.TransfersSkipped-summary.FoldersSkipped,
summary.FoldersSkipped,
summary.TotalBytesTransferred,
summary.TotalBytesSkipped,
summary.PercentComplete, // noted as approx in the format string because won't include in-flight files if this Show command is run from a different process
summary.JobStatus,
)
Expand Down
2 changes: 2 additions & 0 deletions cmd/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -658,6 +658,7 @@ Number of Copy Transfers Completed: %v
Number of Copy Transfers Failed: %v
Number of Deletions at Destination: %v
Total Number of Bytes Transferred: %v
Total Number of Bytes Skipped: %v
Total Number of Bytes Enumerated: %v
Final Job Status: %v%s%s
`,
Expand All @@ -672,6 +673,7 @@ Final Job Status: %v%s%s
summary.TransfersFailed,
cca.atomicDeletionCount,
summary.TotalBytesTransferred,
summary.TotalBytesSkipped,
summary.TotalBytesEnumerated,
summary.JobStatus,
screenStats,
Expand Down
3 changes: 3 additions & 0 deletions common/rpc-models.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,9 @@ type ListJobSummaryResponse struct {
// does not include failed transfers or bytes sent in retries (i.e. no double counting). Includes successful transfers and transfers in progress
TotalBytesTransferred uint64 `json:",string"`

// Includes bytes of data skipped over (i.e. data already exists in blob)
TotalBytesSkipped uint64 `json:",string"`

// sum of the total transfer enumerated so far.
TotalBytesEnumerated uint64 `json:",string"`
// sum of total bytes expected in the job (i.e. based on our current expectation of which files will be successful)
Expand Down
5 changes: 3 additions & 2 deletions jobsAdmin/init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,7 @@ func GetJobSummary(jobID common.JobID) common.ListJobSummaryResponse {
// if no bytes expected, and we should avoid dividing by 0 (which results in NaN)
js.PercentComplete = 100
} else {
js.PercentComplete = 100 * (float32(js.TotalBytesTransferred) / float32(js.TotalBytesExpected))
js.PercentComplete = 100 * (float32(js.TotalBytesTransferred) + float32(js.TotalBytesSkipped)) / float32(js.TotalBytesExpected)
}

// This is added to let FE to continue fetching the Job Progress Summary
Expand Down Expand Up @@ -556,6 +556,7 @@ func resurrectJobSummary(jm ste.IJobMgr) common.ListJobSummaryResponse {
case common.ETransferStatus.SkippedEntityAlreadyExists(),
common.ETransferStatus.SkippedBlobHasSnapshots():
js.TransfersSkipped++
js.TotalBytesSkipped += uint64(jppt.SourceSize)
// getting the source and destination for skipped transfer at position - index
src, dst, isFolder := jpp.TransferSrcDstStrings(t)
js.SkippedTransfers = append(js.SkippedTransfers,
Expand All @@ -575,7 +576,7 @@ func resurrectJobSummary(jm ste.IJobMgr) common.ListJobSummaryResponse {
// if no bytes expected, and we should avoid dividing by 0 (which results in NaN)
js.PercentComplete = 100
} else {
js.PercentComplete = 100 * float32(js.TotalBytesTransferred) / float32(js.TotalBytesExpected)
js.PercentComplete = 100 * (float32(js.TotalBytesTransferred) + float32(js.TotalBytesSkipped)) / float32(js.TotalBytesExpected)
}

// This is added to let FE to continue fetching the Job Progress Summary
Expand Down
1 change: 1 addition & 0 deletions ste/jobStatusManager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ func (jm *jobMgr) handleStatusUpdateMessage() {
}
js.TransfersSkipped++
js.SkippedTransfers = append(js.SkippedTransfers, msg)
js.TotalBytesSkipped += msg.TransferSize
}

case <-jstm.listReq:
Expand Down