new: DataBuffer

fix: MazoviaCharset: overflow

assira/pom.xml

@@ -51,6 +51,7 @@
                                 <include>net/ranides/assira/io/IOEvents*</include>
                                 <include>net/ranides/assira/io/StringInput</include>
                                 <include>net/ranides/assira/io/StringOutput*</include>
+                                <include>net/ranides/assira/io/DataBuffer*</include>
                                 
                                 <include>net/ranides/assira/math/LSFR*</include>
                                 <include>net/ranides/assira/math/MathUtils*</include>

assira/src/main/java/net/ranides/assira/io/DataBuffer.java

@@ -0,0 +1,370 @@
+/*
+ * @author Ranides Atterwim <ranides@gmail.com>
+ * @copyright Ranides Atterwim
+ * @license WTFPL
+ * @url http://ranides.net/projects/assira
+ */
+package net.ranides.assira.io;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.UTFDataFormatException;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CoderResult;
+import net.ranides.assira.collection.arrays.ArrayAllocator;
+import net.ranides.assira.collection.arrays.NativeArray;
+import net.ranides.assira.collection.arrays.NativeArrayAllocator;
+import net.ranides.assira.text.Charsets;
+
+/**
+ *
+ * @author Ranides Atterwim <ranides@gmail.com>
+ */
+public class DataBuffer implements DataInput, DataOutput {
+    
+    // @todo (assira #3) io DataBuffer implements DataInput, DataOutput
+    
+    NativeArray array;
+    ByteBuffer buffer;
+
+    public DataBuffer() {
+        byte[] data = new byte[16];
+        this.array = NativeArray.wrap(data);
+        this.buffer = ByteBuffer.wrap(data);
+    }
+    
+    public DataBuffer seek(int position) throws IOException {
+        if(position != buffer.position(position).position()) {
+            throw new IOException("EOF ?");
+        }
+        return this;
+    }
+    
+    public int seek() {
+        return buffer.position();
+    }
+        
+    private void unget(byte v) {
+        int n =buffer.position();
+        buffer.position(n-1);
+        buffer.put(n-1, v);
+    }
+    
+    @Override
+    public void readFully(byte[] b) throws IOException {
+        readFully(b, 0, b.length);
+    }
+
+    @Override
+    public void readFully(byte[] b, int off, int len) throws IOException {
+        checkEOF(len);
+        buffer.get(b, off, len);
+    }
+
+    @Override
+    public int skipBytes(int n) throws IOException {
+        reserve(n);
+        buffer.position(buffer.position()+n);
+        return n;
+    }
+
+    @Override
+    public boolean readBoolean() throws IOException {
+        checkEOF(1);
+        return buffer.get() != 0;
+    }
+
+    @Override
+    public byte readByte() throws IOException {
+        checkEOF(1);
+        return buffer.get();
+    }
+
+    @Override
+    public int readUnsignedByte() throws IOException {
+        checkEOF(1);
+        return buffer.get() & 0xFF;
+    }
+
+    @Override
+    public short readShort() throws IOException {
+        checkEOF(2);
+        return buffer.getShort();
+    }
+
+    @Override
+    public int readUnsignedShort() throws IOException {
+        checkEOF(2);
+        return buffer.getShort() & 0xFFFF;
+    }
+
+    @Override
+    public char readChar() throws IOException {
+        checkEOF(2);
+        return buffer.getChar();
+    }
+
+    @Override
+    public int readInt() throws IOException {
+        checkEOF(4);
+        return buffer.getInt();
+    }
+
+    @Override
+    public long readLong() throws IOException {
+        checkEOF(8);
+        return buffer.getLong();
+    }
+
+    @Override
+    public float readFloat() throws IOException {
+        checkEOF(4);
+        return buffer.getFloat();
+    }
+
+    @Override
+    public double readDouble() throws IOException {
+        checkEOF(8);
+        return buffer.getDouble();
+    }
+    
+    public String readLine(Charset cs) throws IOException {
+        if(buffer.remaining() <=0) {
+            return null;
+        }
+        CharsetDecoder cd = cs.newDecoder();
+        SBuffer out = new SBuffer();
+
+        while(true) {
+            out.reserve();
+            CharBuffer cb = out.cbuf();
+            int p = buffer.position(); // NOPMD
+            CoderResult cr = cd.decode(buffer, cb, true);
+            if(cr.isError()) {
+                throw new IOException("Encoding error:" + cr.toString());
+            }
+            int end = out.endl();
+            if(end >= 0) {
+                buffer.position(p);
+                cd.decode(buffer, out.cbuf(end), true);
+                return out.toString();
+            }
+            if(cr.isUnderflow()) {
+                out.end = out.position + cb.position();
+                return out.toString();
+            }
+        }
+    }
+    
+    /**
+     * Reads the next line of text from the input stream.
+     * <b>Uses UTF-8 charset</b>. 
+     * <p>
+     * If end of file is encountered
+     * before even one byte can be read, then {@code null}
+     * is returned. Otherwise, bytes are converted to type {@code char}
+     * using UTF-8 charset. If the character {@code '\n'}
+     * is encountered, it is discarded and reading
+     * ceases. If the character {@code '\r'}
+     * is encountered, it is discarded and, if
+     * the following byte converts &#32;to the
+     * character {@code '\n'}, then that is
+     * discarded also; reading then ceases. If
+     * end of file is encountered before either
+     * of the characters {@code '\n'} and
+     * {@code '\r'} is encountered, reading
+     * ceases. Once reading has ceased, a {@code String}
+     * is returned that contains all the characters
+     * read and not discarded, taken in order.
+     * 
+     * @return the next line of text from the input stream,
+     *         or {@code null} if the end of file is
+     *         encountered before a byte can be read.
+     * @exception  IOException  if an I/O error occurs.
+     */
+    @Override
+    public String readLine() throws IOException {
+        return readLine(Charsets.UTF8);
+    }
+
+    @Override
+    public String readUTF() throws IOException {
+        checkEOF(2);
+        int size = readUnsignedShort();
+        byte[] bytes = new byte[size];
+        readFully(bytes);
+        return new String(bytes, Charsets.UTF8);
+
+    }
+    
+    @Override
+    public void write(int b) throws IOException {
+        reserve(1);
+        buffer.put((byte)b);
+    }
+
+    @Override
+    public void write(byte[] b, int off, int len) throws IOException {
+        reserve(len);
+        buffer.put(b, off, len);
+    }
+
+    @Override
+    public void write(byte[] b) throws IOException {
+        reserve(b.length);
+        buffer.put(b);
+    }
+
+    @Override
+    public void writeBoolean(boolean v) throws IOException {
+        reserve(1);
+        buffer.put(v ? (byte)1 : (byte)0);
+    }
+
+    @Override
+    public void writeByte(int v) throws IOException {
+        reserve(1);
+        buffer.put((byte)v);
+    }
+
+    @Override
+    public void writeShort(int v) throws IOException {
+        reserve(2);
+        buffer.putShort((short)v);
+    }
+
+    @Override
+    public void writeChar(int v) throws IOException {
+        reserve(2);
+        buffer.putChar((char)v);
+    }
+
+    @Override
+    public void writeInt(int v) throws IOException {
+        reserve(4);
+        buffer.putInt(v);
+    }
+
+    @Override
+    public void writeLong(long v) throws IOException {
+        reserve(8);
+        buffer.putLong(v);
+    }
+
+    @Override
+    public void writeFloat(float v) throws IOException {
+        reserve(4);
+        buffer.putFloat(v);
+    }
+
+    @Override
+    public void writeDouble(double v) throws IOException {
+        reserve(8);
+        buffer.putDouble(v);
+    }
+
+    @Override
+    public void writeBytes(String s) throws IOException {
+        int len = s.length();
+        reserve(len);
+        for (int i = 0 ; i < len ; i++) {
+            write((byte)s.charAt(i));
+        }
+    }
+
+    @Override
+    public void writeChars(String s) throws IOException {
+        int len = s.length();
+        reserve(2*len);
+        for (int i = 0 ; i < len ; i++) {
+            buffer.putChar(s.charAt(i));
+        }
+    }
+
+    @Override
+    public void writeUTF(String str) throws IOException {
+        byte[] data = str.getBytes(Charsets.UTF8);
+        int size = data.length;
+        if (size > 65535) {
+            throw new UTFDataFormatException("encoded string too long: " + size + " bytes");
+        }
+        
+        reserve(2 + size);
+        buffer.putShort((short)size);
+        buffer.put(data);
+    }
+    
+    private void reserve(int size) {
+        if( buffer.remaining() >= size) {
+            return;
+        }
+        int rem = buffer.capacity() - buffer.position();
+        if( rem >= size) {
+            buffer.limit(buffer.position() + size);
+            return;
+        }
+
+        array = NativeArrayAllocator.grow(array, array.size()+size);
+        ByteBuffer newbuf = ByteBuffer.wrap(array.$array());
+        newbuf.position(buffer.position());
+        newbuf.limit(buffer.position()+size);
+        buffer = newbuf;
+    }
+    
+    private void checkEOF(int size) throws EOFException {
+        if(buffer.remaining() < size) {
+            buffer.position(buffer.limit());
+            throw new EOFException();
+        }
+    }
+    
+    private static class SBuffer {
+        
+        char[] chars = new char[80];
+        int position = 0;
+        int end = -1;
+        
+        CharBuffer cbuf() {
+            return CharBuffer.wrap(chars, position, chars.length-position);
+        }
+        
+        CharBuffer cbuf(int size) {
+            return CharBuffer.wrap(chars, position, size);
+        }
+        
+        void reserve() {
+            chars = ArrayAllocator.ensureCapacity(chars, position+80);
+        }
+        
+        boolean peek(int index, char c) {
+            return (index < (chars.length)) && (chars[index] == c);
+        }
+        
+        int endl() {
+            for(int i=position; i<chars.length; i++) {
+                char c = chars[i];
+                if(c=='\n') {
+                    this.end = i;
+                    return i+1;
+                }
+                if(c=='\r') {
+                    this.end = i;
+                    return peek(i+1, '\n') ? i+2 : i+1;
+                }
+            }
+            return -1;
+        }
+
+        @Override
+        public String toString() {
+            return new String(chars, 0, end);
+        }
+        
+    }
+    
+}

assira/src/main/java/net/ranides/assira/text/MazoviaCharset.java

@@ -1,142 +1,145 @@
-package net.ranides.assira.text;
-
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CharsetEncoder;
-import java.nio.charset.CoderResult;
-import java.nio.charset.spi.CharsetProvider;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-import net.ranides.assira.collection.sets.HashSet;
-
-public class MazoviaCharset extends CharsetProvider {
-
-    private static final String NAME = "mazovia";
-    
-    private static final String[] NAMES = new String[]{
-        "cp-896",
-        "cp896",
-        "cp620",
-        "cp-620",
-        "cp790",
-        "cp-790"
-    };
-    
-    private static final Set<String> NAMES_SET = new HashSet<>(NAMES);
-    
-    private static final List<Charset> CHARSETS = Arrays.asList(new MCharset());
-    
-    private static final char[][] CHARMAP = new char[][]{
-        {'\u0104', 143}//Ą
-      , {'\u0105', 134}//ą
-      , {'\u0106', 149}//Ć
-      , {'\u0107', 141}//ć
-      , {'\u0118', 144}//Ę
-      , {'\u0119', 145}//ę
-      , {'\u0141', 156}//Ł
-      , {'\u0142', 146}//ł
-      , {'\u0143', 165}//Ń
-      , {'\u0144', 164}//ń
-      , {'\u00D3', 163}//Ó
-      , {'\u00F3', 162}//ó
-      , {'\u015A', 152}//Ś
-      , {'\u015B', 158}//ś
-      , {'\u0179', 160}//Ź
-      , {'\u017A', 166}//ź
-      , {'\u017B', 161}//Ż
-      , {'\u017C', 167}//ż
-    };
-
-    @Override
-    public Iterator<Charset> charsets() {
-        return CHARSETS.iterator();
-    }
-
-    @Override
-    public Charset charsetForName(String charsetName) {
-        if(NAME.equalsIgnoreCase(charsetName)){
-            return CHARSETS.get(0);
-        }
-        if (NAMES_SET.contains(charsetName.toLowerCase(Locale.ROOT).trim())) {
-            return CHARSETS.get(0);
-        } 
-        return null;
-    }
-    
-    private static class MCharset extends Charset {
-
-        public MCharset() {
-            super(NAME, NAMES);
-        }
-
-        @Override
-        public boolean contains(Charset cs) {
-            return cs.equals(this);
-        }
-        
-        @Override
-        public CharsetEncoder newEncoder() {
-            return new MEncoder(this);
-        }
-
-        @Override
-        public CharsetDecoder newDecoder() {
-            return new MDecoder(this);
-        }
-
-    }
-    
-    private static class MEncoder extends CharsetEncoder {
-
-        public MEncoder(Charset cs) {
-            super(cs, 1, 1);
-        }
-
-        @Override
-        protected CoderResult encodeLoop(CharBuffer in, ByteBuffer out) {
-            while (in.hasRemaining()) {
-                out.put((byte) (encodeChar(in.get()) & 0xFF));
-            }
-            return CoderResult.UNDERFLOW;
-        }
-
-        private char encodeChar(char c) {
-            for (char[] m : CHARMAP) {
-                if (c == m[0]) {
-                    return m[1];
-                }
-            }
-            return c;
-        }
-    }
-
-    private static class MDecoder extends CharsetDecoder {
-
-        public MDecoder(Charset cs) {
-            super(cs, 1, 1);
-        }
-
-        @Override
-        protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) {
-            while (in.hasRemaining()) {
-                out.put(decodeChar((char) (in.get() & 0x00FF)));
-            }
-            return CoderResult.UNDERFLOW;
-        }
-
-        private char decodeChar(char c) {
-            for (char[] m : CHARMAP) {
-                if (c == m[1]) {
-                    return m[0];
-                }
-            }
-            return c;
-        }
-    }
-
-}
+package net.ranides.assira.text;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.spi.CharsetProvider;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import net.ranides.assira.collection.sets.HashSet;
+
+public class MazoviaCharset extends CharsetProvider {
+
+    private static final String NAME = "mazovia";
+    
+    private static final String[] NAMES = new String[]{
+        "cp-896",
+        "cp896",
+        "cp620",
+        "cp-620",
+        "cp790",
+        "cp-790"
+    };
+    
+    private static final Set<String> NAMES_SET = new HashSet<>(NAMES);
+    
+    private static final List<Charset> CHARSETS = Arrays.asList(new MCharset());
+    
+    private static final char[][] CHARMAP = new char[][]{
+        {'\u0104', 143}//Ą
+      , {'\u0105', 134}//ą
+      , {'\u0106', 149}//Ć
+      , {'\u0107', 141}//ć
+      , {'\u0118', 144}//Ę
+      , {'\u0119', 145}//ę
+      , {'\u0141', 156}//Ł
+      , {'\u0142', 146}//ł
+      , {'\u0143', 165}//Ń
+      , {'\u0144', 164}//ń
+      , {'\u00D3', 163}//Ó
+      , {'\u00F3', 162}//ó
+      , {'\u015A', 152}//Ś
+      , {'\u015B', 158}//ś
+      , {'\u0179', 160}//Ź
+      , {'\u017A', 166}//ź
+      , {'\u017B', 161}//Ż
+      , {'\u017C', 167}//ż
+    };
+
+    @Override
+    public Iterator<Charset> charsets() {
+        return CHARSETS.iterator();
+    }
+
+    @Override
+    public Charset charsetForName(String charsetName) {
+        if(NAME.equalsIgnoreCase(charsetName)){
+            return CHARSETS.get(0);
+        }
+        if (NAMES_SET.contains(charsetName.toLowerCase(Locale.ROOT).trim())) {
+            return CHARSETS.get(0);
+        } 
+        return null;
+    }
+    
+    private static class MCharset extends Charset {
+
+        public MCharset() {
+            super(NAME, NAMES);
+        }
+
+        @Override
+        public boolean contains(Charset cs) {
+            return cs.equals(this);
+        }
+        
+        @Override
+        public CharsetEncoder newEncoder() {
+            return new MEncoder(this);
+        }
+
+        @Override
+        public CharsetDecoder newDecoder() {
+            return new MDecoder(this);
+        }
+
+    }
+    
+    private static class MEncoder extends CharsetEncoder {
+
+        public MEncoder(Charset cs) {
+            super(cs, 1, 1);
+        }
+
+        @Override
+        protected CoderResult encodeLoop(CharBuffer in, ByteBuffer out) {
+            while (in.hasRemaining()) {
+                out.put((byte) (encodeChar(in.get()) & 0xFF));
+            }
+            return CoderResult.UNDERFLOW;
+        }
+
+        private char encodeChar(char c) {
+            for (char[] m : CHARMAP) {
+                if (c == m[0]) {
+                    return m[1];
+                }
+            }
+            return c;
+        }
+    }
+
+    private static class MDecoder extends CharsetDecoder {
+
+        public MDecoder(Charset cs) {
+            super(cs, 1, 1);
+        }
+
+        @Override
+        protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) {
+            while (in.hasRemaining()) {
+                if(out.remaining() < 1) {
+                    return CoderResult.OVERFLOW;
+                }
+                out.put(decodeChar((char) (in.get() & 0x00FF)));
+            }
+            return CoderResult.UNDERFLOW;
+        }
+
+        private char decodeChar(char c) {
+            for (char[] m : CHARMAP) {
+                if (c == m[1]) {
+                    return m[0];
+                }
+            }
+            return c;
+        }
+    }
+
+}

assira/src/test/java/net/ranides/assira/io/DataAdapterTest.java

@@ -28,9 +28,7 @@ import org.junit.Test;
  * @author Ranides Atterwim <ranides@gmail.com>
  */
 public class DataAdapterTest {
-    
-    // @todo (assira #3) io DataBuffer implements DataInput, DataOutput
-    
+   
     @Test
     public void testWriter() {
         TAdapter<Writer, ByteArrayOutputStream> map = new TAdapter<>();

assira/src/test/java/net/ranides/assira/io/DataBufferTest.java

@@ -0,0 +1,103 @@
+/*
+ * @author Ranides Atterwim <ranides@gmail.com>
+ * @copyright Ranides Atterwim
+ * @license WTFPL
+ * @url http://ranides.net/projects/assira
+ */
+package net.ranides.assira.io;
+
+import java.io.IOException;
+import net.ranides.assira.text.Charsets;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Ranides Atterwim <ranides@gmail.com>
+ */
+public class DataBufferTest {
+    
+    @Test
+    public void testWrite() throws IOException {
+        String text = "Quick brown\nfox jumps\nover lazy\ndog\n";
+        
+        DataBuffer db1 = new DataBuffer();
+        
+        db1.write(text.getBytes(Charsets.US_ASCII));
+        db1.write(text.getBytes(Charsets.US_ASCII));
+        db1.write(text.getBytes(Charsets.US_ASCII));
+        
+        byte[] exp = (text+text+text).getBytes(Charsets.US_ASCII);
+        byte[] buf = new byte[exp.length];
+        db1.seek(0);
+        db1.readFully(buf);
+        assertArrayEquals(exp, buf);
+    }
+    
+    @Test
+    public void testReadLine() throws IOException {
+        DataBuffer db1 = new DataBuffer();
+        db1.write("Quick brown\nfox jumps\nover lazy\ndog\n".getBytes(Charsets.US_ASCII));
+        db1.seek(0);
+
+        assertEquals("Quick brown", db1.readLine());
+        assertEquals("fox jumps", db1.readLine());
+        assertEquals("over lazy", db1.readLine());
+        assertEquals("dog", db1.readLine());
+        assertEquals(null, db1.readLine());
+        
+        DataBuffer db2 = new DataBuffer();
+        db2.write("Quick brown\nfox jumps\nover lazy\ndog".getBytes(Charsets.US_ASCII));
+        db2.seek(0);
+        assertEquals("Quick brown", db2.readLine());
+        assertEquals("fox jumps", db2.readLine());
+        assertEquals("over lazy", db2.readLine());
+        assertEquals("dog", db2.readLine());
+        assertEquals(null, db2.readLine());
+        
+        DataBuffer db3 = new DataBuffer();
+        db3.write("Quick brown\rfox jumps\r\nover lazy\n\rdog\r!".getBytes(Charsets.US_ASCII));
+        db3.seek(0);
+        assertEquals("Quick brown", db3.readLine());
+        assertEquals("fox jumps", db3.readLine());
+        assertEquals("over lazy", db3.readLine());
+        assertEquals("", db3.readLine());
+        assertEquals("dog", db3.readLine());
+        assertEquals("!", db3.readLine());
+        assertEquals(null, db3.readLine());
+    }
+
+    @Test
+    public void testReadLineCS() throws IOException {
+        DataBuffer db1 = new DataBuffer();
+        db1.write("Pchnąć w tę\nłódź jeża\nlub ośm\nskrzyń fig\n".getBytes(Charsets.UTF8));
+        db1.seek(0);
+
+        assertEquals("Pchnąć w tę", db1.readLine(Charsets.UTF8));
+        assertEquals("łódź jeża", db1.readLine(Charsets.UTF8));
+        assertEquals("lub ośm", db1.readLine(Charsets.UTF8));
+        assertEquals("skrzyń fig", db1.readLine(Charsets.UTF8));
+        assertEquals(null, db1.readLine(Charsets.UTF8));
+        
+        DataBuffer db2 = new DataBuffer();
+        db2.write("Pchnąć w tę\nłódź jeża\nlub ośm\nskrzyń fig".getBytes(Charsets.UTF8));
+        db2.seek(0);
+        assertEquals("Pchnąć w tę", db2.readLine(Charsets.UTF8));
+        assertEquals("łódź jeża", db2.readLine(Charsets.UTF8));
+        assertEquals("lub ośm", db2.readLine(Charsets.UTF8));
+        assertEquals("skrzyń fig", db2.readLine(Charsets.UTF8));
+        assertEquals(null, db2.readLine(Charsets.UTF8));
+        
+        DataBuffer db3 = new DataBuffer();
+        db3.write("Pchnąć w tę\rłódź jeża\r\nlub ośm\n\rskrzyń fig\r!".getBytes(Charsets.UTF8));
+        db3.seek(0);
+        assertEquals("Pchnąć w tę", db3.readLine(Charsets.UTF8));
+        assertEquals("łódź jeża", db3.readLine(Charsets.UTF8));
+        assertEquals("lub ośm", db3.readLine(Charsets.UTF8));
+        assertEquals("", db3.readLine(Charsets.UTF8));
+        assertEquals("skrzyń fig", db3.readLine(Charsets.UTF8));
+        assertEquals("!", db3.readLine(Charsets.UTF8));
+        assertEquals(null, db3.readLine(Charsets.UTF8));
+    }
+    
+}