Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: event data size limit #212

Merged
merged 5 commits into from
Apr 2, 2024
Merged

feat: event data size limit #212

Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7b3b9af
feat: event data size limit
cpacm Mar 27, 2024
ae7dcdb
feat: event data size limit
cpacm Mar 27, 2024
74cb095
fix: SQLiteBlobTooBigException
cpacm Mar 27, 2024
290f329
remain some space.
cpacm Mar 27, 2024
1f91e4b
add db large data test.
cpacm Apr 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 25 additions & 10 deletions growingio-data/database/src/main/java/com/growingio/android/database/EventDataManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public class EventDataManager {
private static final String TAG = "EventDataManager";

private static final long EVENT_VALID_PERIOD_MILLS = 7L * 24 * 60 * 60_000;
private static final double EVENT_DATA_MAX_SIZE = 2 * 1000 * 1024; // 2M

private final TrackerContext context;
private final String eventsInfoAuthority;
Expand Down Expand Up @@ -93,6 +94,12 @@ Uri insertEvent(GEvent gEvent) {

EventByteArray data = formatData(EventFormatData.format(gEvent));
if (data != null && data.getBodyData() != null) {
if (data.getBodyData().length > EVENT_DATA_MAX_SIZE) {
// SQLiteBlobTooBigException: Row too big to fit into CursorWindow
// cursor window default is 2M, so we should limit the data size
Logger.e(TAG, "event data is too large, ignore it.");
return null;
}
ContentValues contentValues = EventDataTable.putValues(data.getBodyData(), gEvent.getEventType(), gEvent.getSendPolicy());
return contentResolver.insert(uri, contentValues);
}
Expand Down Expand Up @@ -131,23 +138,27 @@ void queryEvents(int policy, int limit, EventDbResult dbResult) {
dbResult.setSuccess(false);
return;
}
long lastId = -1;
List<byte[]> queryList = new ArrayList<>();
ContentProviderClient client = context.getContentResolver().acquireContentProviderClient(eventsInfoAuthority);
try (Cursor cursor = queryEvents(client, policy, limit)) {
int count = 0;
double dataSize = 0;
while (cursor.moveToNext()) {
count++;
String eventType = cursor.getString(cursor.getColumnIndexOrThrow(COLUMN_EVENT_TYPE));
dbResult.setEventType(eventType);
if (cursor.isLast()) {
lastId = cursor.getLong(cursor.getColumnIndexOrThrow(COLUMN_ID));
dbResult.setLastId(lastId);
}
byte[] data = cursor.getBlob(cursor.getColumnIndexOrThrow(COLUMN_DATA));
if (data != null) {
if (data != null && data.length < EVENT_DATA_MAX_SIZE) {
dataSize += data.length;
if (dataSize > EVENT_DATA_MAX_SIZE) {
break;
}
queryList.add(data);
long lastId = cursor.getLong(cursor.getColumnIndexOrThrow(COLUMN_ID));
dbResult.setLastId(lastId);
} else {
Logger.e(TAG, "event data is too large or null, delete it.");
// event data is illegal, delete it.
long delId = cursor.getLong(cursor.getColumnIndexOrThrow(COLUMN_ID));
removeEventById(client, delId);
}
Expand Down Expand Up @@ -182,14 +193,18 @@ void queryEventsAndDelete(int policy, int limit, EventDbResult dbResult) {
List<byte[]> queryList = new ArrayList<>();
try (Cursor cursor = queryEvents(client, policy, limit)) {
int count = 0;
double dataSize = 0;
while (cursor.moveToNext()) {
count++;
if (cursor.isLast()) {
cursor.getLong(cursor.getColumnIndexOrThrow(COLUMN_ID));
}
byte[] data = cursor.getBlob(cursor.getColumnIndexOrThrow(COLUMN_DATA));
if (data != null) {
if (data != null && data.length < EVENT_DATA_MAX_SIZE) {
dataSize += data.length;
if (dataSize > EVENT_DATA_MAX_SIZE) {
break;
}
queryList.add(data);
long lastId = cursor.getLong(cursor.getColumnIndexOrThrow(COLUMN_ID));
dbResult.setLastId(lastId);
}
long delId = cursor.getLong(cursor.getColumnIndexOrThrow(COLUMN_ID));
removeEventById(client, delId);
Expand Down
Loading