All geek questions in one place

Really Long Strings in JSON (> 1 gig) with Jackson Token

I am trying to write JSON processing code with very long string values ​​(more than 1 billion characters long) stored in a file. I do not want to store whole lines in memory (since I can process them in a stream). But I cannot find such an option in Jackson's parser. What I have done so far is a test using the Jackson token offsets (first round of a read file) and a random access file to process lines in a stream (second round of a read file):

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.RandomAccessFile;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.MappingJsonFactory;

public class LongStringJsonTest {
    public static void main(String[] args) throws Exception {
        File tempJson = new File("temp.json");
        PrintWriter pw = new PrintWriter(tempJson);
        pw.print("{\"k1\": {\"k11\": \"");
        for (int i = 0; i < 1e8; i++)
            pw.print("abcdefghij"); 
        pw.print("\"}, \"k2\": \"klmnopqrst\", " +
                "\"k3\": [\"uvwxyz\", \"0123\"]}");
        pw.close();
        searchForStrings(tempJson);
    }

    private static void searchForStrings(File tempJson) throws Exception {
        JsonFactory f = new MappingJsonFactory();
        JsonParser jp = f.createParser(tempJson);
        Map<Long, Long> stringStartToNext = new HashMap<Long, Long>();
        long lastStringStart = -1;
        boolean wasFieldBeforeString = false;
        while (true) {
            JsonToken token = jp.nextToken();
            if (token == null)
                break;
            if (lastStringStart >= 0) {
                stringStartToNext.put(lastStringStart, (wasFieldBeforeString ? -1 : 1) *
                        jp.getTokenLocation().getByteOffset());
                lastStringStart = -1;
                wasFieldBeforeString = false;
            }
            if (token == JsonToken.FIELD_NAME) {
                wasFieldBeforeString = true;
            } else if (token == JsonToken.VALUE_STRING) {
                lastStringStart = jp.getTokenLocation().getByteOffset();
            } else {
                wasFieldBeforeString = false;
            }
        }
        jp.close();
        jp = f.createParser(tempJson);
        RandomAccessFile raf = new RandomAccessFile(tempJson, "r");
        while (true) {
            JsonToken token = jp.nextToken();
            if (token == null)
                break;
            if (token == JsonToken.VALUE_STRING) {
                long start = jp.getTokenLocation().getByteOffset();
                long end = stringStartToNext.get(start);
                // You are able to process stream without keeping all bytes in memory.
                // Here you see strings including quotes around them.
                final long[] length = new long[] {0};
                ByteArrayOutputStream baos = new ByteArrayOutputStream();
                OutputStream os = new OutputStream() {
                    @Override
                    public void write(int b) throws IOException {
                        throw new IOException("Method is not supported");
                    }
                    @Override
                    public void write(byte[] b, int off, int len)
                            throws IOException {
                        if (baos.size() < 20) {
                            baos.write(b, off, Math.min(len, 20));
                            baos.write((int)'.');
                            baos.write((int)'.');
                            baos.write((int)'.');
                        }
                        if (len > 0)
                            length[0] += len;
                    }
                };
                processString(raf, start, end, os);
                String text = new String(baos.toByteArray(), Charset.forName("utf-8"));
                System.out.println("String: " + text + ", length=" + length[0]);
            }
        }
        jp.close();
        raf.close();
    }

    private static void processString(RandomAccessFile raf, long start, long end, 
            OutputStream os) throws Exception {
        boolean wasFieldBeforeString = end < 0;
        int quoteNum = wasFieldBeforeString ? 3 : 1;
        end = Math.abs(end);
        byte[] buffer = new byte[10000];
        raf.seek(start);
        boolean afterBackSlash = false;
        int strLen = (int)(end - start);
        for (int chunk = 0; strLen > 0; chunk++) {
            int ret = raf.read(buffer, 0, Math.min(buffer.length, strLen));
            if (ret < 0)
                break;
            if (ret > 0) {
                int offset = 0;
                if (chunk == 0) {
                    // Assumption that key string doesn't contain double quotes 
                    // and it shorter than buffer size (for simplicity)
                    for (int n = 0; n < quoteNum; n++) {
                        while (true) {
                            if (buffer[offset] == '\"' && !afterBackSlash) {
                                break;
                            } else if (buffer[offset] == '\\') {
                                afterBackSlash = !afterBackSlash;
                            } else {
                                afterBackSlash = false;
                            }
                            offset++;
                        }
                        offset++;
                    }
                    offset--;
                    ret -= offset;
                }
                // Searching for ending quote
                int endQuotePos = offset + (chunk == 0 ? 1 : 0); // Skip open quote
                while (endQuotePos < offset + ret) {
                    if (buffer[endQuotePos] == '\"' && !afterBackSlash) {
                        break;
                    } else if (buffer[endQuotePos] == '\\') {
                        afterBackSlash = !afterBackSlash;
                    } else {
                        afterBackSlash = false;
                    }
                    endQuotePos++;
                }
                if (endQuotePos < offset + ret) {
                    os.write(buffer, offset, endQuotePos + 1 - offset);
                    break;
                }
                os.write(buffer, offset, ret);
                strLen -= ret;
            }
        }
    }
}

This approach does not support Unicode at all. I am curious if there is a way to do this better (or even using some other libs)?

+4
3

, JSON - . - (, JSON , - ). , - . , :

1) FasterXML/Jackson ( ). , , - -, +, unicode .

2) Google/Gson JsonReader, JSON -. nextString (https://github.com/google/gson/blob/master/gson/src/main/java/com/google/gson/stream/JsonReader.java#L816). , JSON ( : https://github.com/google/gson/blob/master/gson/src/main/java/com/google/gson/stream/JsonReader.java#L1317-L1323).

3) fangyidong/Json-simple SAX- push-. : https://github.com/fangyidong/json-simple/blob/master/src/main/java/org/json/simple/parser/ContentHandler.java#L108

4) beckchr/StAXON. JSON XML, XMLStreamReader. , : http://grepcode.com/file/repository.grepcode.com/java/root/jdk/openjdk/6-b14/javax/xml/stream/XMLStreamReader.java#XMLStreamReader.getTextCharacters%28int%2Cchar%5B%5D%2Cint%2Cint%29. , , OutOfMemoryError JSON . :

private static void useStaxon(File tempJson) throws Exception {
    XMLInputFactory factory = new JsonXMLInputFactory();
    XMLStreamReader reader = factory.createXMLStreamReader(new FileReader(tempJson));
    while (true) {
        if (reader.getEventType() == XMLStreamConstants.END_DOCUMENT)
            break;
        if (reader.isCharacters()) {
            long len = reader.getTextLength();
            String text;
            if (len > 20) {
                char[] buffer = new char[20];
                reader.getTextCharacters(0, buffer, 0, buffer.length);
                text = new String(buffer) + "...";
            } else {
                text = reader.getText();
            }
            System.out.println("String: " + text + " (length=" + len + ")");
        }
        reader.next();
    }
    reader.close();
}

:

Exception in thread "main" java.lang.OutOfMemoryError: Java heap space
    at de.odysseus.staxon.json.stream.impl.Yylex.zzRefill(Yylex.java:346)
    at de.odysseus.staxon.json.stream.impl.Yylex.yylex(Yylex.java:600)
    at de.odysseus.staxon.json.stream.impl.Yylex.nextSymbol(Yylex.java:271)
    at de.odysseus.staxon.json.stream.impl.JsonStreamSourceImpl.next(JsonStreamSourceImpl.java:120)
    at de.odysseus.staxon.json.stream.impl.JsonStreamSourceImpl.peek(JsonStreamSourceImpl.java:250)
    at de.odysseus.staxon.json.JsonXMLStreamReader.consume(JsonXMLStreamReader.java:150)
    at de.odysseus.staxon.json.JsonXMLStreamReader.consume(JsonXMLStreamReader.java:153)
    at de.odysseus.staxon.json.JsonXMLStreamReader.consume(JsonXMLStreamReader.java:183)
    at de.odysseus.staxon.json.JsonXMLStreamReader.consume(JsonXMLStreamReader.java:153)
    at de.odysseus.staxon.json.JsonXMLStreamReader.consume(JsonXMLStreamReader.java:183)
    at de.odysseus.staxon.base.AbstractXMLStreamReader.initialize(AbstractXMLStreamReader.java:216)
    at de.odysseus.staxon.json.JsonXMLStreamReader.initialize(JsonXMLStreamReader.java:87)
    at de.odysseus.staxon.json.JsonXMLStreamReader.<init>(JsonXMLStreamReader.java:78)
    at de.odysseus.staxon.json.JsonXMLInputFactory.createXMLStreamReader(JsonXMLInputFactory.java:150)
    at de.odysseus.staxon.json.JsonXMLInputFactory.createXMLStreamReader(JsonXMLInputFactory.java:45)
    at test20150911.LongStringJsonTest.useStaxon(LongStringJsonTest.java:40)
    at test20150911.LongStringJsonTest.main(LongStringJsonTest.java:35)

5) - - , C, JSON BSON. BSON . : https://github.com/dwight/bsontools. , "fromjson" 1 Gb JSON , ( ), - 10 . , 10 1 , ? ( : java ).

, : (1) , , , (2) FasterXML/Jackson, , Java, .

0

, .

JSON, XML CSV, , : , , .

, , , . , / .

. , , , , , .

0

Perhaps this is a valid case when you write your own parser?

JSON parsing should be relatively simple using PushbackReader ().

0
source

Source: https://habr.com/ru/post/1607040/

More articles:


All Articles