From ec0d7e0334c04ce479604638ef6ba8f04f8c556b Mon Sep 17 00:00:00 2001 From: Ivan Mahonin Date: Feb 09 2023 17:13:18 +0000 Subject: neural --- diff --git a/simple/neural/build-nn-heli.sh b/simple/neural/build-nn-heli.sh new file mode 100755 index 0000000..cf50343 --- /dev/null +++ b/simple/neural/build-nn-heli.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +cc -Wall -DNDEBUG -O3 nn-heli.c -lm `pkg-config --cflags --libs helianthus` -o nn-heli + diff --git a/simple/neural/build-nn-trainer.sh b/simple/neural/build-nn-trainer.sh new file mode 100755 index 0000000..953c555 --- /dev/null +++ b/simple/neural/build-nn-trainer.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +cc -Wall -DNDEBUG -O3 nn-trainer.c -lm -o nn-trainer + + + + diff --git a/simple/neural/data/output/.placeholder b/simple/neural/data/output/.placeholder new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/simple/neural/data/output/.placeholder diff --git a/simple/neural/data/symbols-data.bin b/simple/neural/data/symbols-data.bin new file mode 100644 index 0000000..71a355f Binary files /dev/null and b/simple/neural/data/symbols-data.bin differ diff --git a/simple/neural/data/weights.bin b/simple/neural/data/weights.bin new file mode 100644 index 0000000..34f2889 Binary files /dev/null and b/simple/neural/data/weights.bin differ diff --git a/simple/neural/nn-heli.c b/simple/neural/nn-heli.c new file mode 100644 index 0000000..00f75e3 --- /dev/null +++ b/simple/neural/nn-heli.c @@ -0,0 +1,154 @@ + +#include +#include +#include + +#include + +#include "nnlayer.inc.c" + + +NeuralLayer *nl; +Framebuffer fb, fbMin; +Animation fbAnim, fbMinAnim; + +int wasPressed; +double prevX, prevY; + + + +void prepareImage() { + int w, h; + unsigned char *pixels = NULL; + + saveState(); + target(fb); + imageFromViewport(&w, &h, &pixels); + restoreState(); + if (!pixels) return; + + int x0 = w, y0 = h, x1 = 0, y1 = 0; + for(int y = 0; y < h; ++y) { + for(int x = 0; x < w; ++x) { + if (imageGetPixel(w, h, pixels, x, y) != 0x000000ff) { + if (x0 > x) x0 = x; + if (x1 < x) x1 = x; + if (y0 > y) y0 = y; + if (y1 < y) y1 = y; + } + } + } + free(pixels); + pixels = NULL; + + if (x1 < x0 || y1 < y0) return; + + double wx = x1 - x0 + 1; + double wy = y1 - y0 + 1; + double s = 20.0/(wx > wy ? wx : wy); + double cx = (x0 + x1)/2.0; + double cy = (y0 + y1)/2.0; + + double xx = 14 - s*cx; + double yy = 14 - s*cy; + double ww = s*w; + double hh = s*h; + + saveState(); + target(fbMin); + noStroke(); + rectTextured(fbAnim, xx, yy, ww, hh); + imageFromViewport(&w, &h, &pixels); + restoreState(); + + if (!pixels) return; + double *p = nl->a; + for(int y = 0; y < h; ++y) + for(int x = 0; x < w; ++x) + *p++ = colorGetValue(imageGetPixel(w, h, pixels, x, y)); +} + + +void init() { + background(COLOR_BLACK); + stroke(COLOR_WHITE); + fb = createFramebufferEx(512, 512, NULL, FALSE, FALSE, TRUE); + fbMin = createFramebufferEx(28, 28, NULL, FALSE, FALSE, TRUE); + fbAnim = createAnimationFromFramebuffer(fb); + fbMinAnim = createAnimationFromFramebuffer(fbMin); + + saveState(); + target(fb); + clear(); + target(fbMin); + clear(); + restoreState(); + + nl = nlNew(NULL, 784, 0); + nlNew(nl, 256, 0); + nlNew(nl, 128, 0); + nlNew(nl, 64, 0); + nlNew(nl, 64, 0); + nlNew(nl, 10, 0); + nlLoad(nl, "data/weights.bin"); +} + + +void draw() { + saveState(); + + if (mouseDown("left")) { + double x = mouseX(), y = mouseY(); + if (!wasPressed) prevX = x, prevY = y; + + saveState(); + strokeWidth(32); + target(fb); + line(prevX, prevY, x, y); + restoreState(); + + prevX = x, prevY = y; + wasPressed = TRUE; + } else { + wasPressed = FALSE; + } + + if (keyWentDown("space")) { + prepareImage(); + nlPass(nl); + saveState(); + target(fb); + clear(); + restoreState(); + } + + noStroke(); + rectTextured(fbAnim, 0, 0, 512, 512); + + stroke(COLOR_WHITE); + rectTextured(fbMinAnim, 16, 16, 28, 28); + + noFill(); + + NeuralLayer *nlb = nlBack(nl); + textSize(8); + int res = 0; + for(int i = 0; i < 10; ++i) { + if (nlb->a[i] > nlb->a[res]) res = i; + textf(16, 90+8*i, "%d: %lf", i, nlb->a[i]); + } + textSize(16); + textf(16, 60, "%d", res); + + restoreState(); +} + + +int main(int largc, char **largv) { + windowSetVariableFrameRate(); + windowSetInit(&init); + windowSetDraw(&draw); + windowRun(); + return 0; +} + diff --git a/simple/neural/nn-trainer.c b/simple/neural/nn-trainer.c new file mode 100644 index 0000000..0e3bea4 --- /dev/null +++ b/simple/neural/nn-trainer.c @@ -0,0 +1,38 @@ + +#include +#include + +#include "nntrain.inc.c" + + +int main() { + srand(time(NULL)); + + printf("load training data\n"); + NeuralTrainer *nt = ntNewSymbolMap("data/symbols-data.bin", 784, 10); + if (!nt) return 1; + + printf("create neural network\n"); + double tr = 0.1; + NeuralLayer *nl = nlNew(NULL, 784, tr); + nlNew(nl, 256, tr); + nlNew(nl, 128, tr); + nlNew(nl, 64, tr); + nlNew(nl, 64, tr); + nlNew(nl, 10, tr); + + printf("try load previously saved network\n"); + nlLoad(nl, "data/weights.bin"); + + printf("train\n"); + ntTrain(nt, nl, 10, 100, 0.4); + ntTrain(nt, nl, 10, 100, 0.2); + ntTrain(nt, nl, 10, 100, 0.1); + ntTrain(nt, nl, 10, 100, 0.05); + + printf("save neural network weights\n"); + if (!nlSave(nl, "data/output/weights.bin")) return 1; + + return 0; +} + diff --git a/simple/neural/nnlayer.inc.c b/simple/neural/nnlayer.inc.c new file mode 100644 index 0000000..040de48 --- /dev/null +++ b/simple/neural/nnlayer.inc.c @@ -0,0 +1,165 @@ +#ifndef NNLAYER_INC_C +#define NNLAYER_INC_C + + +#include +#include +#include +#include +#include + + + +typedef struct NeuralLayer { + struct NeuralLayer *prev, *next; + int size; + double trainRatio; + double *a, *d, *da, *w; +} NeuralLayer; + + + +NeuralLayer* nlNew(NeuralLayer *prev, int size, double trainRatio) { + assert(size > 0); + while(prev && prev->next) prev = prev->next; + NeuralLayer *nl = calloc(sizeof(NeuralLayer), 1); + nl->size = size; + nl->prev = prev; + nl->trainRatio = trainRatio; + nl->a = calloc(sizeof(*nl->a), size*3); + nl->d = nl->a + size; + nl->da = nl->d + size; + if (prev) { + assert(prev->size > 0); + nl->w = calloc(sizeof(*nl->w), size*prev->size); + for(double *p = nl->w, *e = p + size*prev->size; p < e; ++p) + *p = rand()/(double)RAND_MAX*2 - 1; + prev->next = nl; + } + return nl; +} + + +void nlFree(NeuralLayer *nl) { + if (nl->next) nlFree(nl->next); + if (nl->prev) nl->prev->next = NULL; + free(nl->a); + if (nl->prev) free(nl->w); + free(nl); +} + + +NeuralLayer* nlFront(NeuralLayer *nl) + { while(nl->prev) nl = nl->prev; return nl; } +NeuralLayer* nlBack(NeuralLayer *nl) + { while(nl->next) nl = nl->next; return nl; } + + +int nlToStream(NeuralLayer *nl, FILE *f) { + if (nl->prev) + if (1 != fwrite(nl->w, sizeof(double) * nl->size * nl->prev->size, 1, f)) + return 0; + return nl->next ? nlToStream(nl->next, f) : 1; +} + + +int nlFromStream(NeuralLayer *nl, FILE *f) { + if (nl->prev) + if (1 != fread(nl->w, sizeof(double) * nl->size * nl->prev->size, 1, f)) + return 0; + return nl->next ? nlFromStream(nl->next, f) : 1; +} + + +int nlSave(NeuralLayer *nl, const char *filename) { + FILE *f = fopen(filename, "wb"); + if (!f) + return printf("cannot open file '%s' for write\n", filename), 0; + if (!nlToStream(nl, f)) + return printf("cannot write to file '%s'\n", filename), fclose(f), 0; + fclose(f); + return 1; +} + + +int nlLoad(NeuralLayer *nl, const char *filename) { + FILE *f = fopen(filename, "rb"); + if (!f) + return printf("cannot open file '%s' for read\n", filename), 0; + if (!nlFromStream(nl, f)) + return printf("cannot read from file '%s'\n", filename), fclose(f), 0; + fclose(f); + return 1; +} + + +NeuralLayer* nlPass(NeuralLayer *nl) { + NeuralLayer *nlp = nl->prev; + if (nlp) { + double *nlpa = nlp->a, *ee = nlpa + nlp->size; + double *w = nl->w; + for(double *a = nl->a, *d = nl->d, *e = a + nl->size; a < e; ++a, ++d) { + double s = 0; + for(double *pa = nlpa; pa < ee; ++pa, ++w) + s += *w * *pa; + double ex = exp(-s); + double ex1 = ex + 1; + double ex2 = 1/ex1; + *a = ex2; // sigmoid + *d = ex*ex2*ex2; // sigmoid derivation + } + } + return nl->next ? nlPass(nl->next) : nl; +} + + +NeuralLayer* nlBackpass(NeuralLayer *nl) { + NeuralLayer *nlp = nl->prev; + if (nlp) { + double tr = nl->trainRatio; + double *nlpa = nlp->a, *nlpda = nlp->da, *ee = nlpa + nlp->size; + double *w = nl->w; + if (nlp->prev) { + memset(nlp->da, 0, sizeof(*nlp->da) * nlp->size); + for(double *d = nl->d, *da = nl->da, *e = d + nl->size; d < e; ++d, ++da) { + double ds = *d * *da; + double dst = ds*tr; + for(double *pa = nlpa, *pda = nlpda; pa < ee; ++pa, ++pda, ++w) { + *pda += ds * *w; + *w += dst * *pa; + } + } + } else { + for(double *d = nl->d, *da = nl->da, *e = d + nl->size; d < e; ++d, ++da) { + double dst = *d * *da * tr; + for(double *pa = nlpa; pa < ee; ++pa, ++w) + *w += dst * *pa; + } + } + return nlBackpass(nlp); + } + return nl; +} + + +double nlTrainPass(NeuralLayer *nl, double *x, double *y, double qmin) { + assert(!nl->prev); + double *fa = nl->a; + nl->a = x; + NeuralLayer *nlb = nlPass(nl); + + double qmax = 0; + for(double *a = nlb->a, *da = nlb->da, *e = a + nlb->size; a < e; ++a, ++da, ++y) { + double d = *y - *a; + *da = d; + double q = fabs(d); + if (qmax < q) qmax = q; + } + if (qmax > qmin) nlBackpass(nlb); + + nl->a = fa; + return qmax; +} + + +#endif diff --git a/simple/neural/nntrain.inc.c b/simple/neural/nntrain.inc.c new file mode 100644 index 0000000..7eb97e3 --- /dev/null +++ b/simple/neural/nntrain.inc.c @@ -0,0 +1,144 @@ +#ifndef NNTRAIN_INC_C +#define NNTRAIN_INC_C + + +#include "nnlayer.inc.c" + + +typedef struct NeuralTrainer { + int sizeX, sizeY, count; + double *x, *y; +} NeuralTrainer; + + + +NeuralTrainer* ntNew(int sizeX, int sizeY, int count) { + assert(sizeX > 0); + assert(sizeY > 0); + assert(count > 0); + NeuralTrainer *nt = calloc(sizeof(NeuralTrainer), 1); + nt->sizeX = sizeX; + nt->sizeY = sizeY; + nt->count = count; + nt->x = calloc(sizeof(double)*(sizeX + sizeY)*count, 1); + nt->y = nt->x + sizeX*count; + return nt; +} + + +void ntFree(NeuralTrainer *nt) { + free(nt->x); + free(nt); +} + + +double ntTrain(NeuralTrainer *nt, NeuralLayer *nl, int successCount, int blockSize, double qmin) { + assert(!nl->prev); + assert(nt->sizeX == nl->size); + assert(nt->sizeY == nlBack(nl)->size); + assert(blockSize > 0 && qmin > 0); + + printf("training: %d, %lf\n", blockSize, qmin); + double **blockXY = calloc(sizeof(double)*2, blockSize); + double qmin2 = qmin*0.75; + double qmin3 = qmin2*0.75; + + int success = 0; + int total = 0; + int repeats, blockRepeats; + double qmax0, qsum0, qmax, qsum; + for(int i = 0; i < 10000; ++i) { + for(int i = 0; i < blockSize; ++i) { + int index = rand() % nt->count; + blockXY[i*2 + 0] = nt->x + nt->sizeX*index; + blockXY[i*2 + 1] = nt->y + nt->sizeY*index; + } + + repeats = blockRepeats = 0; + qmax0 = qsum0 = 0; + for(int i = 0; i < 1000; ++i) { + double **xy = blockXY; + qmax = 0, qsum = 0; + for(int i = 0; i < blockSize; ++i, xy += 2) { + double q0 = 0; + for(int i = 0; i < 100; ++i) { + double q = nlTrainPass(nl, xy[0], xy[1], qmin3); + if (!i) q0 = q; + ++repeats; + if (q < qmin3) break; + } + qsum += q0; + if (qmax < q0) qmax = q0; + } + if (!i) { qmax0 = qmax; qsum0 = qsum; } + ++blockRepeats; + if (qmax <= qmin2) break; + } + total += repeats; + + printf(" blocks %d (samples: %d, total: %d, repeats: %3d (%lf)): %lf -> %lf, %lf -> %lf\n", + i+1, (i+1)*blockSize, total, blockRepeats-1, repeats/(double)(blockRepeats*blockSize) - 1, qmax0, qmax, qsum0/blockSize, qsum/blockSize); + + if (qmax0 <= qmin) { + if (++success == successCount) break; + } else { + success = 0; + } + } + + free(blockXY); + printf("done\n"); + return qmax0; +} + + +NeuralTrainer* ntNewSymbolMap(const char *filename, int sizeX, int sizeY) { + FILE *f = fopen(filename, "rb"); + if (!f) + return printf("cannot open file '%s' for read\n", filename), NULL; + fseek(f, 0, SEEK_END); + size_t fs = ftell(f); + fseek(f, 0, SEEK_SET); + + size_t testSize = sizeX + 1; + int count = fs/testSize; + if (!count) + return printf("file '%s' is lesser minimal size\n", filename), fclose(f), NULL; + + unsigned char *data = calloc(testSize, count); + if (count != fread(data, testSize, count, f)) + return printf("cannot read from file '%s'\n", filename), fclose(f), NULL; + + fclose(f); + + NeuralTrainer *nt = ntNew(sizeX, sizeY, count); + const unsigned char *d = data; + double *x = nt->x, *y = nt->y, *ey = y + sizeY*count; + const double delta = 0; + for(double *p = y; p < ey; ++p) *p = delta; + while(y < ey) { + for(double *e = x + sizeX; x < e; ++x, ++d) + *x = *d/255.0; + assert(*d < sizeY); + y[*d++] = 1 - delta; + y += sizeY; + } + return nt; +} + + +void ntPrintSymbol(NeuralTrainer *nt, int index, int width) { + assert(index >= 0 && index < nt->count); + assert(width > 0); + for(int i = 0; i < nt->sizeX; ++i) { + if (i && !(i % width)) printf("\n"); + printf("%c", nt->x[nt->sizeX*index + i] > 0 ? '#' : '.'); + } + printf("\n"); + for(int i = 0; i < nt->sizeY; ++i) + printf(" %4.1lf", nt->y[nt->sizeY*index + i]); + printf("\n"); +} + + +#endif diff --git a/simple/neural/nntrain.inc.c.copy b/simple/neural/nntrain.inc.c.copy new file mode 100644 index 0000000..0482775 --- /dev/null +++ b/simple/neural/nntrain.inc.c.copy @@ -0,0 +1,119 @@ +#ifndef NNTRAIN_INC_C +#define NNTRAIN_INC_C + + +#include "nnlayer.inc.c" + + +typedef struct NeuralTrainer { + int sizeX, sizeY, count; + double *x, *y; +} NeuralTrainer; + + + +NeuralTrainer* ntNew(int sizeX, int sizeY, int count) { + assert(sizeX > 0); + assert(sizeY > 0); + assert(count > 0); + NeuralTrainer *nt = calloc(sizeof(NeuralTrainer), 1); + nt->sizeX = sizeX; + nt->sizeY = sizeY; + nt->count = count; + nt->x = calloc(sizeof(double)*(sizeX + sizeY)*count, 1); + nt->y = nt->x + sizeX*count; + return nt; +} + + +void ntFree(NeuralTrainer *nt) { + free(nt->x); + free(nt); +} + + +double ntTrain(NeuralTrainer *nt, NeuralLayer *nl, int blocks, int blockSize, int repeats, double qmin, double qminSample) { + assert(!nl->prev); + assert(nt->sizeX == nl->size); + assert(nt->sizeY == nlBack(nl)->size); + + printf("training: %d x %d x %d = %d:\n", blocks, blockSize, repeats, blocks*blockSize*repeats); + double qmax, qsum; + int total = 0; + int samples = 0; + for(int i = 0; i < blocks; ++i) { + qmax = qsum = 0; + int repeatsPerBlock = 0; + for(int j = 0; j < blockSize; ++j, ++samples) { + int index = rand() % nt->count; + double *x = nt->x + nt->sizeX*index; + double *y = nt->y + nt->sizeY*index; + double q = 0; + for(int k = 0; k < repeats; ++k) { + double qq = nlTrainPass(nl, x, y); + ++total, ++repeatsPerBlock; + if (!k) q = qq; + if (qq <= qminSample) break; + } + if (qmax < q) qmax = q; + qsum += q; + } + printf(" blocks %d (samples: %d, total: %d (%lf)): %lf, %lf\n", i+1, samples, total, repeatsPerBlock/(double)blockSize - 1, qmax, qsum/blockSize); + if (qmax <= qminSample) break; + } + printf("done\n"); + + return qmax; +} + + +NeuralTrainer* ntNewSymbolMap(const char *filename, int sizeX, int sizeY) { + FILE *f = fopen(filename, "rb"); + if (!f) + return printf("cannot open file '%s' for read\n", filename), NULL; + fseek(f, 0, SEEK_END); + size_t fs = ftell(f); + fseek(f, 0, SEEK_SET); + + size_t testSize = sizeX + 1; + int count = fs/testSize; + if (!count) + return printf("file '%s' is lesser minimal size\n", filename), fclose(f), NULL; + + unsigned char *data = calloc(testSize, count); + if (count != fread(data, testSize, count, f)) + return printf("cannot read from file '%s'\n", filename), fclose(f), NULL; + + fclose(f); + + NeuralTrainer *nt = ntNew(sizeX, sizeY, count); + const unsigned char *d = data; + double *x = nt->x, *y = nt->y, *ey = y + sizeY*count; + const double delta = 0; + for(double *p = y; p < ey; ++p) *p = delta; + while(y < ey) { + for(double *e = x + sizeX; x < e; ++x, ++d) + *x = *d/255.0; + assert(*d < sizeY); + y[*d++] = 1 - delta; + y += sizeY; + } + return nt; +} + + +void ntPrintSymbol(NeuralTrainer *nt, int index, int width) { + assert(index >= 0 && index < nt->count); + assert(width > 0); + for(int i = 0; i < nt->sizeX; ++i) { + if (i && !(i % width)) printf("\n"); + printf("%c", nt->x[nt->sizeX*index + i] > 0 ? '#' : '.'); + } + printf("\n"); + for(int i = 0; i < nt->sizeY; ++i) + printf(" %4.1lf", nt->y[nt->sizeY*index + i]); + printf("\n"); +} + + +#endif