introduce bitloops

src/main/java/puzzle/CsvIndexService.java

@@ -8,7 +8,8 @@ import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.*;
 import java.util.Arrays;
-import java.util.function.Consumer;
+import java.util.function.LongConsumer;
+import static java.nio.charset.StandardCharsets.US_ASCII;
 
 public final class CsvIndexService
         implements Closeable {
@@ -42,7 +43,7 @@ public final class CsvIndexService
       }
       return GSON.fromJson(rawClue, String[].class);
    }
-   public static void lineToLemma(String line, Consumer<Lemma> ok) {
+   public static void lineToLemma(String line, LongConsumer ok) {
       if (line.isBlank()) {
          throw new RuntimeException("Empty line");
       }
@@ -58,7 +59,7 @@ public final class CsvIndexService
          if (Main.VERBOSE) System.err.println("Word too complex: " + line);
          return;
       }
-      ok.accept(new Lemma(id, word));
+      ok.accept(Lemma.pack(id, word.getBytes(US_ASCII)));
    }
    
    public static int simpel(int index) {

src/main/java/puzzle/Export.java

@@ -1,5 +1,7 @@
 package puzzle;
 
+import lombok.Getter;
+import lombok.experimental.Accessors;
 import lombok.experimental.Delegate;
 import puzzle.SwedishGenerator.Dict;
 import puzzle.SwedishGenerator.FillResult;
@@ -96,7 +98,7 @@ public record Export() {
       }
    }
    
-   public record ExportedPuzzle(String[] gridv2, WordOut[] words, int difficulty, Rewards rewards) { }
+   public record ExportedPuzzle(String[] grid, WordOut[] words, int difficulty, Rewards rewards) { }
    
    public record PuzzleResult(SwedishGenerator swe, Dict dict, Gridded mask, FillResult filled) {
       
@@ -215,24 +217,23 @@ public record Export() {
       }
    }
    
-   record DictEntryDTO(ArrayList<Lemma> words, IntListDTO[][] pos) {
+   record DictEntryDTO(LongArrayList words, IntListDTO[][] pos) {
       
       public DictEntryDTO(int L) {
-         this(new ArrayList<>(), new IntListDTO[L][26]);
+         this(new LongArrayList(1024), new IntListDTO[L][26]);
          for (var i = 0; i < L; i++) for (var j = 0; j < 26; j++) pos[i][j] = new IntListDTO();
       }
    }
    
+   @Getter
+   @Accessors(fluent = true)
    static final class IntListDTO {
       
-      int[] a = new int[8];
+      int[] data = new int[8];
-      int   n = 0;
+      int   size = 0;
       void add(int v) {
-         if (n >= a.length) a = Arrays.copyOf(a, a.length * 2);
+         if (size >= data.length) data = Arrays.copyOf(data, data.length * 2);
-         a[n++] = v;
+         data[size++] = v;
       }
-      int size()   { return n; }
-      int[] data() { return a; }
    }
-   
 }

src/main/java/puzzle/LongArrayList.java

@@ -0,0 +1,30 @@
+package puzzle;
+
+import java.util.Arrays;
+public final class LongArrayList {
+   
+   private long[] a;
+   private int    size;
+   
+   public LongArrayList() { this(16); }
+   
+   public LongArrayList(int initialCapacity) {
+      if (initialCapacity < 0) throw new IllegalArgumentException();
+      a = new long[initialCapacity];
+   }
+   
+   public int size() { return size; }
+   
+   public void add(long v) {
+      if (size == a.length) grow();
+      a[size++] = v;
+   }
+   
+   private void grow() {
+      int    newCap = a.length == 0 ? 1 : a.length * 2;
+      long[] n      = new long[newCap];
+      System.arraycopy(a, 0, n, 0, size);
+      a = n;
+   }
+   public long[] toArray() { return Arrays.copyOf(a, this.size); }
+}

src/main/java/puzzle/Main.java

@@ -107,7 +107,7 @@ public class Main {
       printWordsTable(exported.words());
       
       section("Gridv2");
-      for (var row : exported.gridv2()) System.out.println("  " + row);
+      for (var row : exported.grid()) System.out.println("  " + row);
       
       var theme = "algemeen";
       section("Export");
@@ -384,7 +384,7 @@ public class Main {
    
    record JsonExportedPuzzle(String date, String theme, int difficulty, Rewards rewards, String[] grid, WordOut[] words) { }
    private static String toJson(ExportedPuzzle puzzle, String date, String theme) {
-      return CsvIndexService.GSON.toJson(new JsonExportedPuzzle(date, theme, puzzle.difficulty(), puzzle.rewards(), puzzle.gridv2(), puzzle.words()));
+      return CsvIndexService.GSON.toJson(new JsonExportedPuzzle(date, theme, puzzle.difficulty(), puzzle.rewards(), puzzle.grid(), puzzle.words()));
    }
    
    private static String escapeJson(String s) {

src/main/java/puzzle/SwedishGenerator.java

@@ -52,7 +52,7 @@ public record SwedishGenerator(Rng rng) {
    static final int    SIZE                     = C * R;// ~18
    static final int    SIZE_MIN_1               = SIZE - 1;// ~18
    static final double SIZED                    = (double) SIZE;// ~18
-   static final int    TARGET_CLUES             = SIZE >> 2;
+   static final int    TARGET_CLUES             = SIZE >>> 2;
    static final int    MAX_WORD_LENGTH          = C <= R ? C : R;
    static final int    MAX_WORD_LENGTH_PLUS_ONE = MAX_WORD_LENGTH + 1;
    static final int    MIN_LEN                  = Config.MIN_LEN;
@@ -192,19 +192,10 @@ public record SwedishGenerator(Rng rng) {
          else hi &= ~(1L << (idx & 63));
       }
       static boolean isDigit(byte b) { return (b & B48) == B48; }
-      boolean isClue(long index) {
+      boolean isClue(long index)     { return ((index & 64) == 0) ? ((lo >>> index) & 1L) != X : ((hi >>> (index & 63)) & 1L) != X; }
-         if ((index & 64) == 0) return ((lo >> index) & 1L) != X;
+      boolean isClue(int index)      { return ((index & 64) == 0) ? ((lo >>> index) & 1L) != X : ((hi >>> (index & 63)) & 1L) != X; }
-         return ((hi >> (index & 63)) & 1L) != X;
+      boolean notClue(long index)    { return ((index & 64) == 0) ? ((lo >>> index) & 1L) == X : ((hi >>> (index & 63)) & 1L) == X; }
-      }
+      boolean notClue(int index)     { return ((index & 64) == 0) ? ((lo >>> index) & 1L) == X : ((hi >>> (index & 63)) & 1L) == X; }
-      boolean isClue(int index) {
-         if ((index & 64) == 0) return ((lo >> index) & 1L) != 0;
-         return ((hi >> (index & 63)) & 1L) != 0;
-      }
-      boolean notClue(long index) {
-         if ((index & 64) == 0) return ((lo >> index) & 1L) == X;
-         return ((hi >> (index & 63)) & 1L) == X;
-      }
-      boolean notClue(int index) { return ((index & 64) == 0) ? ((lo >> index) & 1L) == X : ((hi >> (index & 63)) & 1L) == X; }
       boolean clueless(int idx) {
          if ((idx & 64) == 0) {
             val test = (1L << idx);
@@ -236,7 +227,7 @@ public record SwedishGenerator(Rng rng) {
       boolean hasRoomForClue(long packed) { return (packed & GT_1_OFFSET_53_BIT) != X && notClue(packed & 0x7FL) && notClue((packed >>> 7) & 0x7FL); }
       void forEachSlot(SlotVisitor visitor) {
          for (var l = lo; l != X; l &= l - 1) processSlot(this, visitor, Long.numberOfTrailingZeros(l));
-         for (var h = hi; h != X; h &= h - 1) processSlot(this, visitor, 64 + Long.numberOfTrailingZeros(h));
+         for (var h = hi; h != X; h &= h - 1) processSlot(this, visitor, 64 | Long.numberOfTrailingZeros(h));
       }
    }
    
@@ -248,9 +239,7 @@ public record SwedishGenerator(Rng rng) {
       static final long INDEX_MASK    = (1L << 24) - 1;     // 24 bits
       static       int  LEMMA_COUNTER = 0;
       static long pack(String word)         { return pack(word.getBytes(US_ASCII)); }
-      static long pack(int index, byte[] b) {
+      static long pack(int index, byte[] b) { return pack(b) | ((long) index << 40); }
-         return pack(b) | ((long) index << 40);
-      }
       static long pack(byte[] b) {
          long w = 0;
          for (var i = 0; i < b.length; i++) w |= ((long) b[i] & ~64) << (i * 5);
@@ -258,14 +247,10 @@ public record SwedishGenerator(Rng rng) {
       }
       public Lemma(int index, String word)   { this(pack(index, word.getBytes(US_ASCII))); }
       public Lemma(String word)              { this(LEMMA_COUNTER++, word); }
-      byte byteAt(int idx)                      { return Lemma.byteAt(word, idx); }// word[]; }
       static byte byteAt(long word, int idx) { return (byte) ((word >>> (idx * 5)) & 0b11111 | B64); }// word[]; }
-      int intAt(int idx)                        { return (int) (((word >>> (idx * 5))) & 0b11111); }// word[]; }
+      static int intAt(long word, int idx)   { return (int) (((word >>> (idx * 5))) & 0b11111); }// word[]; }
-      @Override public int hashCode()           { return unpackIndex(word); }
-      @Override public boolean equals(Object o) { return (o == this) || (o instanceof Lemma l && l.word == word); }
       static String[] clue(long w)           { return CsvIndexService.clues(unpackIndex(w)); }
       static int simpel(long w)              { return CsvIndexService.simpel(unpackIndex(w)); }
-      int length()                              { return Lemma.length(word); }
       static int length(long word) {
          if (word == 0) return 0;
          int highestBit = 63 - Long.numberOfLeadingZeros(word & LETTER_MASK);
@@ -293,25 +278,25 @@ public record SwedishGenerator(Rng rng) {
            DictEntry[] index,
            int length) {
       
-      public Dict(Lemma[] wordz) {
+      public Dict(long[] wordz) {
          var index = new DictEntryDTO[MAX_WORD_LENGTH_PLUS_ONE];
          Arrays.setAll(index, i -> new DictEntryDTO(i));
          for (var lemma : wordz) {
-            var L = lemma.length();
+            var L = Lemma.length(lemma);
             
             var entry = index[L];
             var idx   = entry.words().size();
             entry.words().add(lemma);
             
             for (var i = 0; i < L; i++) {
-               var letter = lemma.intAt(i) - 1;
+               var letter = Lemma.intAt(lemma, i) - 1;
                if (letter < 0 || letter >= 26) throw new RuntimeException("Illegal letter: " + letter + " in word " + lemma);
                entry.pos()[i][letter].add(idx);
             }
          }
          for (int i = MIN_LEN; i < index.length; i++) if (index[i].words().size() <= 0) throw new RuntimeException("No words for length " + i);
          this(Arrays.stream(index).map(i -> {
-                 var words    = i.words().stream().mapToLong(ww -> ww.word).toArray();
+                 var words    = i.words().toArray();
                  int numWords = words.length;
                  int numLongs = (numWords + 63) >>> 6;
                  var bitsets  = new long[i.pos().length * 26][numLongs];
@@ -331,9 +316,9 @@ public record SwedishGenerator(Rng rng) {
       }
       static Dict loadDict(String wordsPath) {
          try {
-            var map = new ArrayList<Lemma>();
+            var map = new LongArrayList(100_000);
             Files.lines(Path.of(wordsPath), UTF_8).forEach(line -> CsvIndexService.lineToLemma(line, map::add));
-            return new Dict(map.toArray(Lemma[]::new));
+            return new Dict(map.toArray());
          } catch (IOException e) {
             e.printStackTrace();
             throw new RuntimeException("Failed to load dictionary from " + wordsPath, e);
@@ -354,7 +339,7 @@ public record SwedishGenerator(Rng rng) {
       public boolean horiz()                             { return horiz(key); }
       public int pos(int i)                              { return offset(packedPos, i); }
       public static boolean horiz(int key)               { return (key & 1) == 0; }
-      public static int offset(long packedPos, int i)    { return (int) ((packedPos >> (i * 7)) & 127); }
+      public static int offset(long packedPos, int i)    { return (int) ((packedPos >>> (i * 7)) & 127); }
       public static int packSlotDir(int idx, int d)      { return (idx << BIT_FOR_DIR) | d; }
    }
    

src/test/java/puzzle/ExportFormatTest.java

@@ -30,9 +30,8 @@ public class ExportFormatTest {
       
       var clueMap = new HashMap<Integer, Long>();
       // key = (cellIndex << 4) | direction
-      var key   = (0 << 4) | 2;
+      var key   = 2;
-      var lemma = new Lemma("TEST");
+      clueMap.put(key, new Lemma("TEST").word());
-      clueMap.put(key, lemma.word());
       
       // Manually fill the grid letters for "TEST" at (0,1), (0,2), (0,3), (0,4)
       grid.setLetter(Grid.offset(0, 1), (byte) 'T');
@@ -74,8 +73,8 @@ public class ExportFormatTest {
       // letterAt only contains letters from placed words.
       // arrow cells are NOT in letterAt unless they are also part of a word (unlikely).
       // So (0,0) should be '#'
-      assertEquals(1, exported.gridv2().length);
+      assertEquals(1, exported.grid().length);
-      assertEquals("#TEST", exported.gridv2()[0]);
+      assertEquals("#TEST", exported.grid()[0]);
    }
    
    @Test
@@ -90,8 +89,8 @@ public class ExportFormatTest {
       assertNotNull(exported);
       assertEquals(0, exported.words().length);
       // Should return full grid with '#'
-      assertEquals(R, exported.gridv2().length);
+      assertEquals(R, exported.grid().length);
-      for (var row : exported.gridv2()) {
+      for (var row : exported.grid()) {
          assertEquals(C, row.length());
          assertTrue(row.matches("#+"));
       }

src/test/java/puzzle/SwedishGeneratorTest.java

@@ -101,20 +101,20 @@ public class SwedishGeneratorTest {
    
    @Test
    void testLemmaAndDict() {
-      var l2a = new Lemma("IN");
+      var l2a = new Lemma("IN").word();
-      var l4a = new Lemma("INER");
+      var l4a = new Lemma("INER").word();
-      var l6a = new Lemma("INEREN");
+      var l6a = new Lemma("INEREN").word();
-      var l7a = new Lemma("INERENA");
+      var l7a = new Lemma("INERENA").word();
-      var l8a = new Lemma("INERENAE");
+      var l8a = new Lemma("INERENAE").word();
       
-      var l1 = new Lemma("APPLE");
+      var l1 = new Lemma("APPLE").word();
-      Assertions.assertEquals(Lemma.pack("APPLE".getBytes(StandardCharsets.US_ASCII)), Lemma.unpackLetters(l1.word()));
+      Assertions.assertEquals(Lemma.pack("APPLE".getBytes(StandardCharsets.US_ASCII)), Lemma.unpackLetters(l1));
-      assertEquals(5, Lemma.length(l1.word()));
+      assertEquals(5, Lemma.length(l1));
-      assertEquals((byte) 'A', l1.byteAt(0));
+      assertEquals((byte) 'A', Lemma.byteAt(l1, 0));
-      assertEquals(1, l1.intAt(0));
+      assertEquals(1, Lemma.intAt(l1, 0));
       
-      var l2   = new Lemma("AXE");
+      var l2   = new Lemma("AXE").word();
-      var dict = new Dict(new Lemma[]{ l1, l2, l2a, l4a, l6a, l7a, l8a });
+      var dict = new Dict(new long[]{ l1, l2, l2a, l4a, l6a, l7a, l8a });
       
       assertEquals(1, dict.index()[3].words().length);
       assertEquals(1, dict.index()[5].words().length);
@@ -213,16 +213,16 @@ public class SwedishGeneratorTest {
    
    @Test
    void testCandidateInfoForPattern() {
-      var l0   = new Lemma("IN");
+      var l0   = new Lemma("IN").word();
-      var l3a  = new Lemma("INE");
+      var l3a  = new Lemma("INE").word();
-      var l4a  = new Lemma("INER");
+      var l4a  = new Lemma("INER").word();
-      var l6a  = new Lemma("INEREN");
+      var l6a  = new Lemma("INEREN").word();
-      var l7a  = new Lemma("INERENA");
+      var l7a  = new Lemma("INERENA").word();
-      var l8a  = new Lemma("INERENAE");
+      var l8a  = new Lemma("INERENAE").word();
-      var l1   = new Lemma("APPLE");
+      var l1   = new Lemma("APPLE").word();
-      var l2   = new Lemma("APPLY");
+      var l2   = new Lemma("APPLY").word();
-      var l3   = new Lemma("BANAN");
+      var l3   = new Lemma("BANAN").word();
-      var dict = new Dict(new Lemma[]{ l0, l1, l2, l3, l3a, l4a, l6a, l7a, l8a });
+      var dict = new Dict(new long[]{ l0, l1, l2, l3, l3a, l4a, l6a, l7a, l8a });
       
       // Pattern "APP--" for length 5
       var context = new Context();