From 5c75ada63298e879363e8289f52f33dab9e94cc9 Mon Sep 17 00:00:00 2001 From: likeazir Date: Mon, 11 Nov 2024 17:05:12 +0100 Subject: [PATCH 1/5] rudimentary pagination fix to keep emoji size under 1MB cursor limit --- .../api/session/AccountSessionManager.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java b/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java index 77a00a557..feff147f1 100644 --- a/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java +++ b/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java @@ -382,7 +382,7 @@ public class AccountSessionManager{ } private void readInstanceInfo(SQLiteDatabase db, Set domains){ - try(Cursor cursor=db.query("instances", null, "`domain` IN ("+String.join(", ", Collections.nCopies(domains.size(), "?"))+")", domains.toArray(new String[0]), null, null, null)){ + try(Cursor cursor=db.query("instances", new String[]{"domain", "instance_obj", "last_updated", "version"}, "`domain` IN ("+String.join(", ", Collections.nCopies(domains.size(), "?"))+")", domains.toArray(new String[0]), null, null, null)){ ContentValues values=new ContentValues(); while(cursor.moveToNext()){ DatabaseUtils.cursorRowToContentValues(cursor, values); @@ -393,11 +393,28 @@ public class AccountSessionManager{ case 2 -> InstanceV2.class; default -> throw new IllegalStateException("Unexpected value: " + version); }); - List emojis=MastodonAPIController.gson.fromJson(values.getAsString("emojis"), new TypeToken>(){}.getType()); + //poor man's pagination / terrible performance + StringBuilder emojiString = new StringBuilder(); + try(Cursor emojiCountCursor=db.rawQuery("select length(emojis) from instances where `domain` = ?", new String[]{domain})){ + emojiCountCursor.moveToNext(); + int totalEmostringLen = emojiCountCursor.getInt(0); + //max cursor size is 1MB, stay at 0.8MB to prevent crash + int pagesize = 800000; + for (int i = 0; i < (totalEmostringLen / 800000) + 1; i++){ + try(Cursor emojiCursor=db.rawQuery("select substr(emojis,?, ?) from instances where `domain` = ?", new String[]{String.valueOf(i * pagesize), String.valueOf(pagesize),domain})){ + emojiCursor.moveToNext(); + emojiString.append(emojiCursor.getString(0)); + } + } + } + List emojis=MastodonAPIController.gson.fromJson(emojiString.toString(), new TypeToken>(){}.getType()); instances.put(domain, instance); customEmojis.put(domain, groupCustomEmojis(emojis)); instancesLastUpdated.put(domain, values.getAsLong("last_updated")); } + } catch(Exception ex) { + Log.d(TAG, "readInstanceInfo failed", ex); + return; } if(!loadedInstances){ loadedInstances=true; @@ -639,6 +656,7 @@ public class AccountSessionManager{ } private static class DatabaseHelper extends SQLiteOpenHelper{ + public DatabaseHelper(){ super(MastodonApp.context, "accounts.db", null, DB_VERSION); } From 4b5f84d78106c5b9127b31e0d25a8e686fd19ad3 Mon Sep 17 00:00:00 2001 From: likeazir Date: Mon, 11 Nov 2024 17:18:15 +0100 Subject: [PATCH 2/5] remove stray newline --- .../joinmastodon/android/api/session/AccountSessionManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java b/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java index feff147f1..aeac3c6c1 100644 --- a/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java +++ b/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java @@ -656,7 +656,6 @@ public class AccountSessionManager{ } private static class DatabaseHelper extends SQLiteOpenHelper{ - public DatabaseHelper(){ super(MastodonApp.context, "accounts.db", null, DB_VERSION); } From 86e369201a3c5714bde70bb7abada86c2ed44597 Mon Sep 17 00:00:00 2001 From: likeazir Date: Wed, 13 Nov 2024 17:32:46 +0100 Subject: [PATCH 3/5] remove extra query to get emoji row length --- .../api/session/AccountSessionManager.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java b/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java index aeac3c6c1..00f474be3 100644 --- a/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java +++ b/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java @@ -382,7 +382,9 @@ public class AccountSessionManager{ } private void readInstanceInfo(SQLiteDatabase db, Set domains){ - try(Cursor cursor=db.query("instances", new String[]{"domain", "instance_obj", "last_updated", "version"}, "`domain` IN ("+String.join(", ", Collections.nCopies(domains.size(), "?"))+")", domains.toArray(new String[0]), null, null, null)){ + //limit emoji string to 0.5MB characters to stay under 1MB cursor limit + try(Cursor cursor=db.rawQuery("SELECT domain, instance_obj, substring(emojis,0,500000) AS emojis, length(emojis) AS emojiLength, last_updated, version FROM instances WHERE `domain` IN (" + + String.join(", ", "?".repeat(domains.size())) + ")", domains.toArray(new String[0]))) { ContentValues values=new ContentValues(); while(cursor.moveToNext()){ DatabaseUtils.cursorRowToContentValues(cursor, values); @@ -393,21 +395,20 @@ public class AccountSessionManager{ case 2 -> InstanceV2.class; default -> throw new IllegalStateException("Unexpected value: " + version); }); - //poor man's pagination / terrible performance - StringBuilder emojiString = new StringBuilder(); - try(Cursor emojiCountCursor=db.rawQuery("select length(emojis) from instances where `domain` = ?", new String[]{domain})){ - emojiCountCursor.moveToNext(); - int totalEmostringLen = emojiCountCursor.getInt(0); - //max cursor size is 1MB, stay at 0.8MB to prevent crash - int pagesize = 800000; - for (int i = 0; i < (totalEmostringLen / 800000) + 1; i++){ - try(Cursor emojiCursor=db.rawQuery("select substr(emojis,?, ?) from instances where `domain` = ?", new String[]{String.valueOf(i * pagesize), String.valueOf(pagesize),domain})){ + StringBuilder emojiSB = new StringBuilder(); + emojiSB.append(values.getAsString("emojis")); + //get emoji in chunks of 1MB if it didn't fit in the first query + int emojiStringLength = values.getAsInteger("emojiLength"); + if(emojiStringLength > 500000){ + int pagesize=1000000; + for(int start = 500000; start < emojiStringLength; start += pagesize){ + try(Cursor emojiCursor=db.rawQuery("select substr(emojis,?, ?) from instances where `domain` = ?", new String[]{String.valueOf(start), String.valueOf(pagesize), domain})){ emojiCursor.moveToNext(); - emojiString.append(emojiCursor.getString(0)); + emojiSB.append(emojiCursor.getString(0)); } } } - List emojis=MastodonAPIController.gson.fromJson(emojiString.toString(), new TypeToken>(){}.getType()); + List emojis=MastodonAPIController.gson.fromJson(emojiSB.toString(), new TypeToken>(){}.getType()); instances.put(domain, instance); customEmojis.put(domain, groupCustomEmojis(emojis)); instancesLastUpdated.put(domain, values.getAsLong("last_updated")); From 6779b5cc4399e799f94337839f8fdde82bf02cef Mon Sep 17 00:00:00 2001 From: likeazir Date: Wed, 13 Nov 2024 17:49:30 +0100 Subject: [PATCH 4/5] single queries for domains to prevent running into the cursor limit --- .../api/session/AccountSessionManager.java | 61 ++++++++++--------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java b/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java index 00f474be3..c5b206780 100644 --- a/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java +++ b/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java @@ -382,40 +382,41 @@ public class AccountSessionManager{ } private void readInstanceInfo(SQLiteDatabase db, Set domains){ - //limit emoji string to 0.5MB characters to stay under 1MB cursor limit - try(Cursor cursor=db.rawQuery("SELECT domain, instance_obj, substring(emojis,0,500000) AS emojis, length(emojis) AS emojiLength, last_updated, version FROM instances WHERE `domain` IN (" - + String.join(", ", "?".repeat(domains.size())) + ")", domains.toArray(new String[0]))) { - ContentValues values=new ContentValues(); - while(cursor.moveToNext()){ - DatabaseUtils.cursorRowToContentValues(cursor, values); - String domain=values.getAsString("domain"); - int version=values.getAsInteger("version"); - Instance instance=MastodonAPIController.gson.fromJson(values.getAsString("instance_obj"), switch(version){ - case 1 -> InstanceV1.class; - case 2 -> InstanceV2.class; - default -> throw new IllegalStateException("Unexpected value: " + version); - }); - StringBuilder emojiSB = new StringBuilder(); - emojiSB.append(values.getAsString("emojis")); - //get emoji in chunks of 1MB if it didn't fit in the first query - int emojiStringLength = values.getAsInteger("emojiLength"); - if(emojiStringLength > 500000){ - int pagesize=1000000; - for(int start = 500000; start < emojiStringLength; start += pagesize){ - try(Cursor emojiCursor=db.rawQuery("select substr(emojis,?, ?) from instances where `domain` = ?", new String[]{String.valueOf(start), String.valueOf(pagesize), domain})){ - emojiCursor.moveToNext(); - emojiSB.append(emojiCursor.getString(0)); + for(String domain : domains){ + final int maxEmojiLength=500000; + try(Cursor cursor=db.rawQuery("SELECT domain, instance_obj, substring(emojis,0,?) AS emojis, length(emojis) AS emojiLength, last_updated, version FROM instances WHERE `domain` = ?", + new String[]{String.valueOf(maxEmojiLength) , domain})) { + ContentValues values=new ContentValues(); + while(cursor.moveToNext()){ + DatabaseUtils.cursorRowToContentValues(cursor, values); + int version=values.getAsInteger("version"); + Instance instance=MastodonAPIController.gson.fromJson(values.getAsString("instance_obj"), switch(version){ + case 1 -> InstanceV1.class; + case 2 -> InstanceV2.class; + default -> throw new IllegalStateException("Unexpected value: "+version); + }); + StringBuilder emojiSB=new StringBuilder(); + emojiSB.append(values.getAsString("emojis")); + //get emoji in chunks of 1MB if it didn't fit in the first query + int emojiStringLength=values.getAsInteger("emojiLength"); + if(emojiStringLength>maxEmojiLength){ + final int pagesize=1000000; + for(int start=maxEmojiLength; start emojis=MastodonAPIController.gson.fromJson(emojiSB.toString(), new TypeToken>(){}.getType()); + instances.put(domain, instance); + customEmojis.put(domain, groupCustomEmojis(emojis)); + instancesLastUpdated.put(domain, values.getAsLong("last_updated")); } - List emojis=MastodonAPIController.gson.fromJson(emojiSB.toString(), new TypeToken>(){}.getType()); - instances.put(domain, instance); - customEmojis.put(domain, groupCustomEmojis(emojis)); - instancesLastUpdated.put(domain, values.getAsLong("last_updated")); + }catch(Exception ex){ + Log.d(TAG, "readInstanceInfo failed", ex); + return; } - } catch(Exception ex) { - Log.d(TAG, "readInstanceInfo failed", ex); - return; } if(!loadedInstances){ loadedInstances=true; From be82274bee8e036012a387ba9d872c2bdb4b5250 Mon Sep 17 00:00:00 2001 From: likeazir Date: Wed, 13 Nov 2024 17:52:01 +0100 Subject: [PATCH 5/5] consistent naming --- .../android/api/session/AccountSessionManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java b/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java index c5b206780..ea896dc36 100644 --- a/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java +++ b/mastodon/src/main/java/org/joinmastodon/android/api/session/AccountSessionManager.java @@ -384,7 +384,7 @@ public class AccountSessionManager{ private void readInstanceInfo(SQLiteDatabase db, Set domains){ for(String domain : domains){ final int maxEmojiLength=500000; - try(Cursor cursor=db.rawQuery("SELECT domain, instance_obj, substring(emojis,0,?) AS emojis, length(emojis) AS emojiLength, last_updated, version FROM instances WHERE `domain` = ?", + try(Cursor cursor=db.rawQuery("SELECT domain, instance_obj, substring(emojis,0,?) AS emojis, length(emojis) AS emoji_length, last_updated, version FROM instances WHERE `domain` = ?", new String[]{String.valueOf(maxEmojiLength) , domain})) { ContentValues values=new ContentValues(); while(cursor.moveToNext()){ @@ -398,7 +398,7 @@ public class AccountSessionManager{ StringBuilder emojiSB=new StringBuilder(); emojiSB.append(values.getAsString("emojis")); //get emoji in chunks of 1MB if it didn't fit in the first query - int emojiStringLength=values.getAsInteger("emojiLength"); + int emojiStringLength=values.getAsInteger("emoji_length"); if(emojiStringLength>maxEmojiLength){ final int pagesize=1000000; for(int start=maxEmojiLength; start