Gather data

This commit is contained in:
mike
2026-01-10 01:40:42 +01:00
parent b698b9e8be
commit 7cd40c90be
3 changed files with 10 additions and 10 deletions

View File

@@ -219,7 +219,6 @@ public record SwedishGenerator(Rng rng) {
static int LEMMA_COUNTER = 0;
public Lemma(int index, String word, int simpel, String[] clu) { this(index, word.getBytes(StandardCharsets.US_ASCII), simpel, clu); }
public Lemma(String word, int simpel, String clue) { this(LEMMA_COUNTER++, word, simpel, new String[]{ clue }); }
public Lemma(String word, int simpel, String[] clue) { this(LEMMA_COUNTER++, word, simpel, clue); }
byte byteAt(int idx) { return word[idx]; }
@Override public int hashCode() { return index; }
@Override public boolean equals(Object o) { return (o == this) || (o instanceof Lemma l && l.index == index); }
@@ -260,7 +259,7 @@ public record SwedishGenerator(Rng rng) {
raw = Files.readString(Path.of(wordsPath), StandardCharsets.UTF_8);
} catch (IOException e) {
e.printStackTrace();
raw = "WOORD,level_1_to_10,hint\nEU,2,hint\nUUR,2,hint\nAUTO,2,hint\nBOOM,2,hint\nHUIS,2,hint\nKAT,2,hint\nZEE,2,hint\nRODE,2,hint\nDRAAD,2,hint\nKENNIS,2,hint\nNETWERK,2,hint\nPAKTE,2,hint\n";
throw new RuntimeException("Failed to load dictionary from " + wordsPath, e);
}
var map = new ArrayList<Lemma>();
@@ -270,8 +269,9 @@ public record SwedishGenerator(Rng rng) {
System.err.println("Empty line: " + line);
continue;
}
var parts = line.split(",", 4);
var word = parts[0].trim();
var parts = line.split(",", 5);
var id = Integer.parseInt(parts[0].trim());
var word = parts[1].trim();
if (first && word.equalsIgnoreCase("WOORD")) {
first = false;
continue;
@@ -284,17 +284,17 @@ public record SwedishGenerator(Rng rng) {
}
// CSV has level 1-10. llmScores use 10-level.
int score = 10 - Integer.parseInt(parts[1].trim());
int score = Integer.parseInt(parts[2].trim());
if (score < 1) {
if (Main.VERBOSE) System.err.println("Word too complex: " + line);
continue;
}
int simpel = Integer.parseInt(parts[2].trim());
var rawClue = parts[3].trim();
int simpel = Integer.parseInt(parts[3].trim());
var rawClue = parts[4].trim();
if (rawClue.startsWith("\"") && rawClue.endsWith("\"")) {
rawClue = rawClue.substring(1, rawClue.length() - 1).replace("\"\"", "\"");
}
map.add(new Lemma(s, simpel, GSON.fromJson(rawClue, String[].class)));
map.add(new Lemma(id,s, simpel, GSON.fromJson(rawClue, String[].class)));
}
return new Dict(map.toArray(Lemma[]::new));