Skip to content

Instantly share code, notes, and snippets.

@courville
Created January 3, 2020 07:53
Show Gist options
  • Save courville/4c45ba5cded3387146c2497ff7bccfb7 to your computer and use it in GitHub Desktop.
Save courville/4c45ba5cded3387146c2497ff7bccfb7 to your computer and use it in GitHub Desktop.
fight Row too big to fit into CursorWindow
diff --git a/src/com/archos/mediascraper/AutoScrapeService.java b/src/com/archos/mediascraper/AutoScrapeService.java
index 256242c..4115703 100644
--- a/src/com/archos/mediascraper/AutoScrapeService.java
+++ b/src/com/archos/mediascraper/AutoScrapeService.java
@@ -22,8 +22,12 @@ import android.app.Service;
import android.content.ContentValues;
import android.content.Context;
import android.content.Intent;
+import android.database.AbstractWindowedCursor;
import android.database.ContentObserver;
+import android.database.CrossProcessCursorWrapper;
import android.database.Cursor;
+import android.database.CursorWindow;
+import android.database.sqlite.SQLiteCursor;
import android.net.Uri;
import android.os.Binder;
import android.os.Build;
@@ -315,6 +319,19 @@ public class AutoScrapeService extends Service {
// find all videos not scraped yet looking at VideoStore.Video.VideoColumns.ARCHOS_MEDIA_SCRAPER_ID
Cursor cursor = getFileListCursor(shouldRescrapAll&&onlyNotFound ?PARAM_SCRAPED_NOT_FOUND:shouldRescrapAll?PARAM_ALL:PARAM_NOT_SCRAPED);
+ // to fight against SQLiteBlobTooBigException: Row too big to fit into CursorWindow
+ // cf. https://stackoverflow.com/questions/51959944/sqliteblobtoobigexception-row-too-big-to-fit-into-cursorwindow-requiredpos-0-t
+ // cf. https://medium.com/androiddevelopers/large-database-queries-on-android-cb043ae626e8
+ // another way would be https://readyandroid.wordpress.com/query-huge-database-in-android-cursor-size-is-limited-to-1mb/
+ // note: this could happen also in VideoStoreImportService
+ if (Build.VERSION.SDK_INT >= 28) {
+ // disable the ⅓ of a window heuristic
+ ((SQLiteCursor)((CrossProcessCursorWrapper)cursor).getWrappedCursor()).setFillWindowForwardOnly(false);
+ // configure CursorWindow size, by default is it 2MB?
+ CursorWindow cw = new CursorWindow("AutoScrapeService", 5000);
+ AbstractWindowedCursor ac = (AbstractWindowedCursor) cursor;
+ ac.setWindow(cw);
+ }
if(DBG) Log.d(TAG, "startScraping: (re)starting thread number of files identified to process " + cursor.getCount());
NfoWriter.ExportContext exportContext = null;
if (NfoWriter.isNfoAutoExportEnabled(AutoScrapeService.this))
@@ -509,6 +526,7 @@ public class AutoScrapeService extends Service {
+ ", remaining=" + sNumberOfFilesRemainingToProcess + ", mNetworkOrScrapErrors=" + mNetworkOrScrapErrors +
", sNumberOfFilesNotScraped=" + sNumberOfFilesNotScraped);
+ // TODO: MARC this crashes on moveToNext if big collection scraping
} while (cursor.moveToNext() && isEnable(AutoScrapeService.this));
sIsScraping = false;
if(cursor.getCount() == mNetworkOrScrapErrors) { //when as many errors, we assume we don't have the internet or that the scraper returns an error, do not loop
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment