Skip to content

Commit

Permalink
#173: Ao10-performance-Optimized-Mediacanner
Browse files Browse the repository at this point in the history
  • Loading branch information
k3b committed Mar 8, 2021
1 parent f80863b commit c955404
Show file tree
Hide file tree
Showing 10 changed files with 299 additions and 54 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@
* Created by k3b on 14.07.2015.
*/
public class AndroFotoFinderApp extends Application {
private static String fileNamePrefix = "androFotofinder.logcat-";
private static final String fileNamePrefix = "androFotofinder.logcat-";

public static MediaContent2DBUpdateService getMediaContent2DbUpdateService() {
return MediaContent2DBUpdateService.instance;
Expand Down Expand Up @@ -123,7 +123,7 @@ private static void registerMediaContentProvider(Context context, IMediaReposito
// switching from mediaImageDbReplacement to Contentprovider
MediaContent2DBUpdateService.instance.clearMediaCopy();
}
FotoSql.setMediaDBApi(mediaContentproviderRepository);
FotoSql.setMediaDBApi(mediaContentproviderRepository, null);
MediaContent2DBUpdateService.instance = null;
}

Expand All @@ -141,7 +141,7 @@ private static IMediaRepositoryApi registerAo10MediaImageDbReplacement(Context c

// read from copy database, write to both: copy-database and content-provider
final MergedMediaRepository mediaDBApi = new MergedMediaRepository(mediaDBRepository, mediaContentproviderRepository);
FotoSql.setMediaDBApi(mediaDBApi);
FotoSql.setMediaDBApi(mediaDBApi, mediaDBRepository);

MediaContent2DBUpdateService.instance = new MediaContent2DBUpdateService(context, writableDatabase);

Expand All @@ -151,13 +151,14 @@ private static IMediaRepositoryApi registerAo10MediaImageDbReplacement(Context c
}

PhotoChangeNotifyer.registerContentObserver(context, GlobalMediaContentObserver.getInstance(context));

return mediaDBApi;
} catch (RuntimeException ignore) {
Log.w(Global.LOG_CONTEXT,
"Cannot open Database (missing permissions) "
+ DatabaseHelper.getDatabasePath(context) + " "
+ ignore.getMessage(), ignore);
FotoSql.setMediaDBApi(new MediaDBRepositoryLoadOnDemand(context));
FotoSql.setMediaDBApi(new MediaDBRepositoryLoadOnDemand(context), null);
}
return null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@
import de.k3b.geo.io.GeoUri;
import de.k3b.io.GalleryFilterParameter;
import de.k3b.io.IDirectory;
import de.k3b.io.IProgessListener;
import de.k3b.io.StringUtils;
import de.k3b.io.XmpFile;
import de.k3b.io.collections.SelectedFiles;
Expand Down Expand Up @@ -201,8 +202,9 @@ private static int updateIncompleteMediaDatabase(String debugPrefix, Context con
Log.d(Global.LOG_CONTEXT, message.toString());
}

IProgessListener progessListener = context instanceof IProgessListener ? ((IProgessListener) context) : null;
PhotoPropertiesMediaFilesScannerAsyncTask scanner = new PhotoPropertiesMediaFilesScannerAsyncTask(
PhotoPropertiesMediaFilesScanner.getInstance(context), context, why);
PhotoPropertiesMediaFilesScanner.getInstance(context), context, why, progessListener);
scanner.execute(null, missing.toArray(new IFile[missing.size()]));
return missing.size();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,8 @@ public class FotoSql extends FotoSqlBase {

private static IMediaRepositoryApi mediaDBApi;

private static IMediaRepositoryApi mediaLocalDatabase;

public static IMediaRepositoryApi getMediaDBApi() {
if ((firstRun) && (FotoSql.mediaDBApi != null)) {
firstRun = false;
Expand All @@ -300,8 +302,13 @@ public static IMediaRepositoryApi getMediaDBApi() {
return FotoSql.mediaDBApi;
}

public static void setMediaDBApi(IMediaRepositoryApi mediaDBApi) {
public static IMediaRepositoryApi getMediaLocalDatabase() {
return mediaLocalDatabase;
}

public static void setMediaDBApi(IMediaRepositoryApi mediaDBApi, IMediaRepositoryApi mediaLocalDatabase) {
FotoSql.mediaDBApi = mediaDBApi;
FotoSql.mediaLocalDatabase = mediaLocalDatabase;
}

public static final double getGroupFactor(final double _zoomLevel) {
Expand Down Expand Up @@ -1099,6 +1106,17 @@ public static int deleteMedia(String dbgContext, List<String> pathsToBeRemoved,
return 0;
}

public static int deleteMedia(IMediaRepositoryApi mediaDBApi,
String dbgContext,
List<Long> idsToBeRemoved,
boolean preventDeleteImageFile) {
if ((idsToBeRemoved != null) && (idsToBeRemoved.size() > 0)) {
String whereDelete = SQL_COL_PK + " in (" + ListUtils.toString(",", idsToBeRemoved) + ")";
return mediaDBApi.deleteMedia(dbgContext, whereDelete, null, preventDeleteImageFile);
}
return 0;
}

public static int deleteMediaWithNullPath() {
/// delete where SQL_COL_PATH + " is null" throws null pointer exception
QueryParameter wherePathIsNull = new QueryParameter();
Expand Down
20 changes: 17 additions & 3 deletions app/src/main/java/de/k3b/android/androFotoFinder/tagDB/TagSql.java
Original file line number Diff line number Diff line change
Expand Up @@ -267,15 +267,29 @@ public static void addWhereTagsIncluded(QueryParameter resultQuery, List<String>
}
}

public static QueryParameter createQueryIdPathDateForMediaScan(Date dateLastAddedOrNull) {
QueryParameter query = new QueryParameter()
.addColumn(TagSql.SQL_COL_PK, TagSql.SQL_COL_PATH, SQL_COL_DATE_ADDED)
.addFrom(TagSql.SQL_TABLE_EXTERNAL_CONTENT_URI_FILE_NAME)
.addWhere("( " + SQL_COL_EXT_XMP_LAST_MODIFIED_DATE + " is null or " + SQL_COL_EXT_XMP_LAST_MODIFIED_DATE + " = 0)" +
" or (" + SQL_COL_LAT + " is null or " + SQL_COL_LAT + " = 0) " +
" or (" + SQL_COL_EXT_TAGS + " is null or " + SQL_COL_EXT_TAGS + " = '')")
.addOrderBy(SQL_COL_DATE_ADDED + " asc");
if (dateLastAddedOrNull != null) {
addWhereDateAddedMinMax(query, dateLastAddedOrNull.getTime(), 0);
}
return query;
}

public static int fixPrivate() {
// update ... set media_type=1001 where media_type=1 and tags like '%;PRIVATE;%'
ContentValues values = new ContentValues();
values.put(SQL_COL_EXT_MEDIA_TYPE, MEDIA_TYPE_IMAGE_PRIVATE);
StringBuilder where = new StringBuilder();
where
.append(TagSql.FILTER_EXPR_PUBLIC)
.append(" AND (")
.append(TagSql.FILTER_EXPR_TAGS_INCLUDED);
.append(TagSql.FILTER_EXPR_PUBLIC)
.append(" AND (")
.append(TagSql.FILTER_EXPR_TAGS_INCLUDED);
if (LibGlobal.renamePrivateJpg) {
where.append(" OR ").append(TagSql.FILTER_EXPR_PATH_LIKE.replace("?","'%" +
PhotoPropertiesUtil.IMG_TYPE_PRIVATE + "'"));
Expand Down
13 changes: 8 additions & 5 deletions app/src/main/java/de/k3b/android/io/AndroidFileCommands.java
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import de.k3b.android.util.PhotoChangeNotifyer;
import de.k3b.android.util.PhotoPropertiesMediaFilesScanner;
import de.k3b.android.util.RecursivePhotoPropertiesMediaFilesScannerAsyncTask;
import de.k3b.android.util.ScannerTaskFactory;
import de.k3b.android.widget.FilePermissionActivity;
import de.k3b.database.QueryParameter;
import de.k3b.io.DirectoryFormatter;
Expand Down Expand Up @@ -399,11 +400,13 @@ private void onMediaScannerAnswer(Activity activity, IFile scanRootDir,

final String message = activity.getString(R.string.scanner_menu_title);

final RecursivePhotoPropertiesMediaFilesScannerAsyncTask scanner = (RecursivePhotoPropertiesMediaFilesScannerAsyncTask.sScanner != null)
? RecursivePhotoPropertiesMediaFilesScannerAsyncTask.sScanner :
new RecursivePhotoPropertiesMediaFilesScannerAsyncTask(
mScanner, activity, message,
fullScan, rescanNeverScannedByAPM, scanForDeleted);
IProgessListener progessListener = activity instanceof IProgessListener ? ((IProgessListener) activity) : null;
final RecursivePhotoPropertiesMediaFilesScannerAsyncTask scanner =
(RecursivePhotoPropertiesMediaFilesScannerAsyncTask.sScanner != null)
? RecursivePhotoPropertiesMediaFilesScannerAsyncTask.sScanner :
ScannerTaskFactory.createScannerTask(message, mScanner,
fullScan, rescanNeverScannedByAPM, scanForDeleted, progessListener);

synchronized (this) {
if (RecursivePhotoPropertiesMediaFilesScannerAsyncTask.sScanner == null) {
RecursivePhotoPropertiesMediaFilesScannerAsyncTask.sScanner = scanner;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/*
* Copyright (c) 2021 by k3b.
*
* This file is part of AndroFotoFinder.
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <http://www.gnu.org/licenses/>
*/

package de.k3b.android.util;

import android.content.Context;
import android.content.SharedPreferences;
import android.database.Cursor;
import android.preference.PreferenceManager;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import de.k3b.android.androFotoFinder.queries.FotoSql;
import de.k3b.android.androFotoFinder.queries.IMediaRepositoryApi;
import de.k3b.android.androFotoFinder.tagDB.TagSql;
import de.k3b.database.QueryParameter;
import de.k3b.io.IProgessListener;
import de.k3b.io.filefacade.FileFacade;
import de.k3b.io.filefacade.IFile;

public class Ao10DbUpdateOnlyPhotoPropertiesMediaFilesScannerAsyncTask extends RecursivePhotoPropertiesMediaFilesScannerAsyncTask {
private static final String AO_10_NEW_SCANN_LAST_DATE_ADDED = "ao10NewScannLastDateAdded";
private final IMediaRepositoryApi mediaDBApi;
private final Date dateLastAdded;

public Ao10DbUpdateOnlyPhotoPropertiesMediaFilesScannerAsyncTask(
IMediaRepositoryApi mediaDBApi,
PhotoPropertiesMediaFilesScanner scanner,
Context context, String why,
Date dateLastAdded,
IProgessListener progessListener) {
super(scanner, context, why, false, false, false, progessListener);
this.mediaDBApi = mediaDBApi;
this.dateLastAdded = dateLastAdded;
}

public static void saveDateLastAdded(Context context, Date date) {
SharedPreferences sharedPref = PreferenceManager.getDefaultSharedPreferences(context);
SharedPreferences.Editor editor = sharedPref.edit();
if (date != null) {
editor.putLong(AO_10_NEW_SCANN_LAST_DATE_ADDED, date.getTime());
} else {
editor.remove(AO_10_NEW_SCANN_LAST_DATE_ADDED);
}
editor.commit();
}

public static Date loadDateLastAdded(Context context) {
SharedPreferences sharedPref = PreferenceManager.getDefaultSharedPreferences(context);
long result = sharedPref.getLong(AO_10_NEW_SCANN_LAST_DATE_ADDED, 0);
if (result == 0) return null;
return new Date(result);
}

@Override
protected Integer doInBackground(IFile[]... pathNames) {
// do not call super.doInBackground here because logic is different
String dbgContext = "First APM-Re-Scan Photos";
int resultCount = 0;
boolean oldValue = this.mScanner.setIgnoreNoMediaCheck(true);
StringBuilder outDebugLog = new StringBuilder();
List<Long> notFound = new ArrayList<>();
long dateAddedInSecs = 0;
try {
onProgress(0, 0, "#");
QueryParameter query = TagSql.createQueryIdPathDateForMediaScan(dateLastAdded);
Cursor c = mediaDBApi.createCursorForQuery(outDebugLog, dbgContext, query, null, null);
int size = c.getCount();
if (c.moveToFirst()) {
do {
long id = c.getLong(0);
String path = c.getString(1);
dateAddedInSecs = (!c.isNull(2)) ? c.getLong(2) : 0;
IFile file = FileFacade.convert(dbgContext, path);
int modifyCount = mScanner.updateAndroid42(mediaDBApi, dbgContext, id, file);
if (modifyCount == 0) notFound.add(id);
this.mFileCount++;
this.mCurrentFolder = path;
if (mFileCount % 10 == 1) {
onProgress(mFileCount, size, path);
}
} while (c.moveToNext());

}
} finally {
this.mScanner.setIgnoreNoMediaCheck(oldValue);
if (notFound.size() > 0) {
FotoSql.deleteMedia(mediaDBApi, dbgContext + " not found", notFound, false);
}
if (dateAddedInSecs != 0) {
saveDateLastAdded(mContext, new Date(dateAddedInSecs * 1000));
}
}
return resultCount;
}

// PhotoPropertiesMediaFilesScanner
}
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,8 @@ public abstract class PhotoPropertiesMediaFilesScanner {

private final Map<String, Boolean> noMediaCache = new HashMap<>();

private boolean ignoreNoMediaCheck = false;

public PhotoPropertiesMediaFilesScanner(Context context) {
mContext = context.getApplicationContext();
}
Expand Down Expand Up @@ -259,7 +261,7 @@ private int excludeNomediaFiles(IFile[] fullPathNames) {
for (int i = 0; i < fullPathNames.length; i++) {
IFile fullPathName = fullPathNames[i];
if (fullPathName != null) {
if (!PhotoPropertiesUtil.isImage(fullPathName, PhotoPropertiesUtil.IMG_TYPE_ALL) || isNoMedia(fullPathName, 5, this.noMediaCache)) {
if (!PhotoPropertiesUtil.isImage(fullPathName, PhotoPropertiesUtil.IMG_TYPE_ALL) || excludeIsNoMedia(fullPathName)) {
fullPathNames[i] = null;
} else {
itemsLeft++;
Expand All @@ -271,6 +273,17 @@ private int excludeNomediaFiles(IFile[] fullPathNames) {
return itemsLeft;
}

public boolean setIgnoreNoMediaCheck(boolean ignoreNoMediaCheck) {
boolean oldValue = this.ignoreNoMediaCheck;
this.ignoreNoMediaCheck = ignoreNoMediaCheck;
return oldValue;
}

protected boolean excludeIsNoMedia(IFile fullPathName) {
if (ignoreNoMediaCheck) return false;
return isNoMedia(fullPathName, 5, this.noMediaCache);
}

private int insertIntoMediaDatabase(IFile[] newPathNames) {
int modifyCount = 0;

Expand All @@ -286,8 +299,10 @@ private int insertIntoMediaDatabase(IFile[] newPathNames) {
Long id = inMediaDb.get(fileName.getAbsolutePath());
if (id != null) {
// already exists
modifyCount += updateAndroid42("PhotoPropertiesMediaFilesScanner.insertIntoMediaDatabase already existing "
, id, fileName);
modifyCount += updateAndroid42(
getMediaDBApi(),
"PhotoPropertiesMediaFilesScanner.insertIntoMediaDatabase already existing ",
id, fileName);
} else {
modifyCount += insertAndroid42(
"PhotoPropertiesMediaFilesScanner.insertIntoMediaDatabase new item ",
Expand Down Expand Up @@ -522,11 +537,11 @@ private static void setFieldIfNeccessary(ContentValues values, String fieldName,
}
}

private int updateAndroid42(String dbgContext, long id, IFile file) {
protected int updateAndroid42(IMediaRepositoryApi mediaDBApi, String dbgContext, long id, IFile file) {
if ((file != null) && file.exists() && file.canRead()) {
ContentValues values = createDefaultContentValues();
getExifFromFile(values, file);
return getMediaDBApi().execUpdate(dbgContext, id, values);
return mediaDBApi.execUpdate(dbgContext, id, values);
}
return 0;
}
Expand Down
Loading

0 comments on commit c955404

Please sign in to comment.