Skip to content

Commit

Permalink
Explore the new RNAcentral microservice to export data
Browse files Browse the repository at this point in the history
  • Loading branch information
@carlosribas
carlosribas committed Jul 8, 2024
1 parent 39b79ee commit b9ce27c
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 42 deletions.
1 change: 1 addition & 0 deletions rnacentral/portal/static/js/components/routes.service.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ angular.module("routes", []).service('routes', ['$interpolate', function($interp
quickGoSummaryPage: 'https://www.ebi.ac.uk/QuickGO/term/{{ term_id }}',
quickGoChart: 'https://www.ebi.ac.uk/QuickGO/services/ontology/{{ ontology }}/terms/{{ term_ids }}/chart?base64=true',
qcStatusApi: '/api/v1/rna/{{ upi }}/qc-status/{{ taxid }}',
exportApp: 'http://hx-rke-wp-webadmin-12-worker-3.caas.ebi.ac.uk:31867',
};


Expand Down
96 changes: 81 additions & 15 deletions rnacentral/portal/static/js/components/text-search/export-results.search.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,21 @@ limitations under the License.
* Angular.js code for exporting metadata search results.
*/

;angular.module('textSearch').controller('ExportResultsCtrl', ['$scope', '$location', '$http', '$interval', '$window', function($scope, $location, $http, $interval, $window) {
;angular.module('textSearch').controller('ExportResultsCtrl', ['$scope', '$location', '$http', '$interval', '$window', 'routes', function($scope, $location, $http, $interval, $window, routes) {

$scope.export = {
query: null,
job_id: null,
data_type: null,
status: null,
hits: 0,
hits: null,
progress: -1,
downloadUrl: null,
ended_at: null,
enqueued_at: null,
error_message: '',
};
$scope.routes = routes;

var interval;

Expand All @@ -35,19 +38,40 @@ limitations under the License.
*/
function get_job_status() {
return $http({
url: '/export/job-status?job=' + $scope.export.job_id,
url: routes.exportApp() + '/download/' + $scope.export.job_id + '/' + $scope.export.data_type,
method: 'GET'
}).then(
function(response) {
$scope.export = _.extend($scope.export, response.data);
$scope.export.expiration = new Date($scope.export.expiration);
if (response.data.status === 'finished' || response.data.status === 'failed') {
if (response.headers('content-type').includes('application/json')) {
$scope.export.hits = response.data.hit_count;

// Check progress
if ($scope.export.data_type === 'fasta') {
$scope.export.progress = (response.data.progress_ids + response.data.progress_fasta) / 2 || 0;
} else if ($scope.export.data_type === 'json') {
$scope.export.progress = (response.data.progress_ids + response.data.progress_db_data) / 2 || 0;
} else {
$scope.export.progress = response.data.progress_ids || 0;
}

// Check status
$scope.export.status = response.data.state === 'RUNNING' ? 'running' : response.data.state ? response.data.state : 'pending';
if ($scope.export.status === 'FAILURE') {
$interval.cancel(interval);
}
} else {
$interval.cancel(interval);
$scope.export.status = 'finished';
$scope.export.progress = 100;

// Store the download URL
var blob = new Blob([response.data], { type: response.headers('content-type') });
$scope.export.downloadUrl = window.URL.createObjectURL(blob);
}
update_page_title();
},
function(response) {
if ( response.status === 404 ) {
if (response.status === 404) {
$scope.export.error_message = 'Job not found';
} else {
$scope.export.error_message = 'Unknown error';
Expand All @@ -56,7 +80,28 @@ limitations under the License.
update_page_title();
}
);
}
}

/**
* Download the file.
*/
function fetch_file() {
return $http({
url: routes.exportApp() + '/download/' + $scope.export.job_id + '/' + $scope.export.data_type,
method: 'GET',
responseType: 'arraybuffer' // Ensure the response is handled as binary for file download
}).then(
function(response) {
var blob = new Blob([response.data], { type: response.headers('content-type') });
$scope.export.downloadUrl = window.URL.createObjectURL(blob); // Create the download URL
$scope.downloadFile(); // Trigger the download
},
function(response) {
$scope.export.error_message = 'Error downloading file';
update_page_title();
}
);
}

/**
* Format progress for use with ng-style and CSS.
Expand All @@ -70,20 +115,18 @@ limitations under the License.
* Set polling interval dynamically based on the number of hits.
*/
function poll_job_status() {
var min_interval = 1, // 1 second
max_interval = 3, // 3 seconds
polling_interval = Math.max(min_interval, Math.min($scope.export.hits / 1000, max_interval)) * 1000;
var max_interval = 3000; // 3 seconds

interval = $interval(function(){
get_job_status();
}, polling_interval);
}, max_interval);
}

/**
* Show progress in page title.
*/
function update_page_title() {
if ($scope.export.status === 'failed') {
if ($scope.export.status === 'FAILURE') {
$window.document.title = 'Export failed';
} else if ($scope.export.error_message !== '') {
$window.document.title = 'Results expired';
Expand All @@ -92,16 +135,39 @@ limitations under the License.
}
}

/**
* Function to handle file download via button.
*/
$scope.downloadFile = function() {
if ($scope.export.downloadUrl) {
var a = document.createElement('a');
a.style.display = 'none';
a.href = $scope.export.downloadUrl;
a.download = $scope.export.job_id + '.' + ($scope.export.data_type === 'json' ? 'json.gz' : $scope.export.data_type === 'fasta' ? 'fasta.gz' : 'txt.gz');
document.body.appendChild(a);
a.click();
window.URL.revokeObjectURL($scope.export.downloadUrl);
}
};

/**
* Fetch the file for download.
*/
$scope.triggerDownload = function() {
fetch_file();
};

/**
* Get job id from the url and begin retrieving data.
*/
function initialize() {
if ($location.url().indexOf("/export/results?job=") > -1) {
if ($location.url().indexOf("/export/results?job=") > -1) {
$scope.export.job_id = $location.search().job;
$scope.export.data_type = $location.search().data_type;
get_job_status().then(function(){
poll_job_status();
});
}
}
}

// get job id and start updating the status
Expand Down
18 changes: 16 additions & 2 deletions ...tal/static/js/components/text-search/text-search-results/text-search-results.component.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,10 +355,24 @@ var textSearchResults = {
* - open the results page in a new window.
*/
ctrl.exportResults = function(format) {
$http.get(ctrl.routes.submitQuery() + '?q=' + search.preprocessQuery(ctrl.search.query) + '&format=' + format).then(
var queryParam = encodeURIComponent(`(${search.preprocessQuery(ctrl.search.query)})`);
var apiUrl = `https://www.ebi.ac.uk/ebisearch/ws/rest/rnacentral?query=${queryParam}&size=1000&sort=id&format=json`;

var payload = {
api_url: apiUrl,
data_type: format
};

$http.post(routes.exportApp() + '/fetch-data/', JSON.stringify(payload), {
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
}).then(
function(response) {
ctrl.showExportError = false;
window.location.href = ctrl.routes.resultsPage() + '?job=' + response.data.job_id;
var task_id = response.data.task_id;
window.location.href = ctrl.routes.resultsPage() + '?job=' + task_id + '&data_type=' + format;
},
function(response) {
ctrl.showExportError = true;
Expand Down
2 changes: 1 addition & 1 deletion ...tral/portal/static/js/components/text-search/text-search-results/text-search-results.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ <h1 style="margin-bottom: 0px">
<ul class="dropdown-menu pull-right" role="menu">
<li><a href="" ng-click="$ctrl.exportResults('fasta')">FASTA</a></li>
<li><a href="" ng-click="$ctrl.exportResults('json')">JSON</a></li>
<li><a href="" ng-click="$ctrl.exportResults('list')">RNAcentral ids</a></li>
<li><a href="" ng-click="$ctrl.exportResults('txt')">RNAcentral ids</a></li>
</ul>
</div>
</small>
Expand Down
31 changes: 7 additions & 24 deletions rnacentral/portal/templates/portal/search/export-job-results.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,18 +37,17 @@ <h1>
<div class="row">
<div class="col-md-12">

<div class="alert alert-info col-md-6" role="alert" ng-show="(export.status === 'queued' || export.status === 'started') && export.error_message === ''">
<div class="alert alert-info cidsol-md-6" role="alert" ng-show="(export.status === 'pending' || export.status === 'running') && export.error_message === ''">
<i class="fa fa-spinner fa-spin fa-2x pull-left"></i>
Your query has been submitted and the results will become available here shortly.
</div>

<div class="alert alert-success col-md-6" role="alert" ng-show="export.status === 'finished'">
<i class="fa fa-check-circle fa-3x pull-left"></i>
The results are available for download and will be kept
until {{ export.expiration | date:'MMMM d'}}.
The results are available for download and will be kept for up to 7 days.
</div>

<div class="alert alert-danger col-md-9" role="alert" ng-show="export.status === 'failed'">
<div class="alert alert-danger col-md-9" role="alert" ng-show="export.error_message === 'Unknown error'">
<i class="fa fa-warning fa-3x pull-left"></i>
<p>
There was a problem while exporting the search results.
Expand All @@ -58,7 +57,7 @@ <h1>
</p>
</div>

<div class="alert alert-danger col-md-8" role="alert" ng-show="export.error_message !== '' && export.status !== 'failed'">
<div class="alert alert-danger col-md-8" role="alert" ng-show="export.error_message === 'Job not found'">
<i class="fa fa-warning fa-3x pull-left"></i>
<p>
The results might have expired.
Expand All @@ -82,34 +81,18 @@ <h1>
<div class="row" ng-show="export.progress >= 0">
<div class="col-md-6">
<ul>
<li class="margin-top-5px">Exporting <strong>{{ export.hits | number }}</strong> sequences in <strong>{{ export.format }}</strong> format</li>
<li class="margin-top-5px">
Query: <code>{{ export.query }}</code>
<small><a href="/search?q={{ export.query }}" target="_blank">Modify</a> <span class="text-muted">(in a new window)</span></small>
</li>
<li class="margin-top-5px">Status: <strong ng-class="{'text-danger': export.status === 'failed', 'text-info': export.status === 'queued', 'text-info': export.status === 'started', 'text-success': export.status === 'finished'}">{{ export.status }}</strong></li>
<li class="margin-top-5px">Exporting <strong>{{ export.hits }}</strong> sequences in <strong>{{ export.data_type }}</strong> format</li>
<li class="margin-top-5px">Status: <strong ng-class="{'text-danger': export.status === 'failed', 'text-info': export.status === 'running', 'text-info': export.status === 'pending', 'text-success': export.status === 'finished'}">{{ export.status }}</strong></li>
</ul>

<div ng-show="export.status === 'finished'" >
<hr>
<a target="_self" class="btn btn-primary btn-lg" href="{% endverbatim %}{% url 'export-download-result' %}{% verbatim %}?job={{ export.job_id }}"><i class="fa fa-download"></i> Download results</a>
<a target="_self" class="btn btn-primary btn-lg" ng-click="triggerDownload()"><i class="fa fa-download"></i> Download results</a>
</div>

</div>
</div>

<div class="row" ng-show="export.hits == 250000">
<div class="col-md-12">
<br>
<div class="alert alert-warning col-md-6" role="alert">
<p>
The maximum number of sequences that can be exported using the online interface is {{ export.hits | number }}.
Please <a href="" onclick="$('.doorbell-feedback').click()">let us know</a> if you need to export more data.
</p>
</div>
</div>
</div>

</div> <!-- cloak -->

</div>
Expand Down

0 comments on commit b9ce27c

Please sign in to comment.