|
|
036a8f |
#ifndef TRAIN_INC_CPP
|
|
|
036a8f |
#define TRAIN_INC_CPP
|
|
|
e865c9 |
|
|
|
e865c9 |
|
|
|
e865c9 |
#include <chrono></chrono>
|
|
|
e865c9 |
#include <thread></thread>
|
|
|
e865c9 |
|
|
|
e865c9 |
|
|
|
e865c9 |
#include "layer.inc.cpp"
|
|
|
e865c9 |
|
|
|
e865c9 |
|
|
|
e865c9 |
long long timeUs() {
|
|
|
e865c9 |
static std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
|
|
|
e865c9 |
return (long long)std::chrono::duration_cast<std::chrono::microseconds>( std::chrono::steady_clock::now() - begin ).count();</std::chrono::microseconds>
|
|
|
e865c9 |
}
|
|
|
e865c9 |
|
|
|
e865c9 |
|
|
|
8e5348 |
struct Quality {
|
|
|
8e5348 |
AccumReal train;
|
|
|
8e5348 |
AccumReal human;
|
|
|
8e5348 |
|
|
|
8e5348 |
inline Quality& operator+=(const Quality &b) {
|
|
|
8e5348 |
train += b.train;
|
|
|
8e5348 |
human += b.human;
|
|
|
8e5348 |
return *this;
|
|
|
8e5348 |
}
|
|
|
8e5348 |
|
|
|
8e5348 |
inline Quality& operator*=(AccumReal x) {
|
|
|
8e5348 |
train *= x;
|
|
|
8e5348 |
human *= x;
|
|
|
8e5348 |
return *this;
|
|
|
8e5348 |
}
|
|
|
8e5348 |
|
|
|
8e5348 |
inline bool operator<(const Quality &b) const {
|
|
|
8e5348 |
return human < b.human ? true
|
|
|
8e5348 |
: b.human < human ? false
|
|
|
8e5348 |
: train < b.train;
|
|
|
8e5348 |
}
|
|
|
8e5348 |
};
|
|
|
8e5348 |
|
|
|
8e5348 |
|
|
|
e865c9 |
class Trainer {
|
|
|
e865c9 |
private:
|
|
|
e865c9 |
std::atomic<unsigned int=""> barrierCounter;</unsigned>
|
|
|
8e5348 |
std::vector<quality> qualities;</quality>
|
|
|
e865c9 |
|
|
|
e865c9 |
public:
|
|
|
e865c9 |
Layer *layer;
|
|
|
e865c9 |
AccumReal ratio;
|
|
|
e865c9 |
int threadsCount;
|
|
|
e865c9 |
int itersPerBlock;
|
|
|
e865c9 |
int blocksPerSaving;
|
|
|
e865c9 |
int blocksCount;
|
|
|
e865c9 |
AccumReal qmin;
|
|
|
e865c9 |
|
|
|
e865c9 |
protected:
|
|
|
8e5348 |
std::atomic<unsigned int=""> skipBackpass;</unsigned>
|
|
|
e865c9 |
Layer *fl;
|
|
|
e865c9 |
Layer *bl;
|
|
|
e865c9 |
|
|
|
e865c9 |
virtual bool prepare() { return true; }
|
|
|
e865c9 |
virtual bool prepareBlock() { return true; }
|
|
|
e865c9 |
virtual void finishBlock() { }
|
|
|
e865c9 |
virtual void finish() { }
|
|
|
e865c9 |
|
|
|
e865c9 |
virtual void loadData(Barrier &barrier, int block, int iter) { }
|
|
|
8e5348 |
virtual Quality verifyData(Barrier &barrier, int block, int iter) { return Quality{}; }
|
|
|
e865c9 |
|
|
|
e865c9 |
private:
|
|
|
e865c9 |
void threadFunc(int tid, int block) {
|
|
|
e865c9 |
Barrier barrier(barrierCounter, tid, threadsCount);
|
|
|
e865c9 |
|
|
|
8e5348 |
Quality sumQ = {};
|
|
|
e865c9 |
for(int i = 0; i < itersPerBlock; ++i) {
|
|
|
e865c9 |
barrier.wait();
|
|
|
e865c9 |
loadData(barrier, block, i);
|
|
|
e865c9 |
|
|
|
e865c9 |
for(Layer *l = fl->next; l; l = l->next) {
|
|
|
e865c9 |
barrier.wait();
|
|
|
e865c9 |
l->pass(barrier);
|
|
|
e865c9 |
}
|
|
|
e865c9 |
|
|
|
e865c9 |
barrier.wait();
|
|
|
8e5348 |
skipBackpass = 0;
|
|
|
e865c9 |
sumQ += verifyData(barrier, block, i);
|
|
|
8e5348 |
|
|
|
e865c9 |
barrier.wait();
|
|
|
8e5348 |
bool skipBp = skipBackpass;
|
|
|
e865c9 |
|
|
|
e865c9 |
barrier.wait();
|
|
|
8e5348 |
if (ratio > 0 && !skipBp) {
|
|
|
e865c9 |
for(Layer *l = bl; l->prev && l->prev->prev; l = l->prev) {
|
|
|
e865c9 |
barrier.wait();
|
|
|
e865c9 |
l->backpassDeltas(barrier);
|
|
|
e865c9 |
}
|
|
|
e865c9 |
for(Layer *l = bl; l->prev; l = l->prev) {
|
|
|
e865c9 |
barrier.wait();
|
|
|
e865c9 |
l->backpassWeights(barrier);
|
|
|
e865c9 |
}
|
|
|
e865c9 |
}
|
|
|
e865c9 |
}
|
|
|
8e5348 |
qualities[tid] = sumQ;
|
|
|
e865c9 |
}
|
|
|
e865c9 |
|
|
|
e865c9 |
|
|
|
8e5348 |
Quality runThreads(int block) {
|
|
|
e865c9 |
barrierCounter = 0;
|
|
|
e865c9 |
std::vector<std::thread*> t(threadsCount, nullptr);</std::thread*>
|
|
|
e865c9 |
for(int i = 1; i < threadsCount; ++i)
|
|
|
e865c9 |
t[i] = new std::thread(&Trainer::threadFunc, this, i, block);
|
|
|
e865c9 |
threadFunc(0, block);
|
|
|
e865c9 |
|
|
|
8e5348 |
Quality result = qualities[0];
|
|
|
e865c9 |
for(int i = 1; i < threadsCount; ++i)
|
|
|
e865c9 |
{ t[i]->join(); delete t[i]; result += qualities[i]; }
|
|
|
8e5348 |
return result *= 1/(AccumReal)itersPerBlock;
|
|
|
e865c9 |
}
|
|
|
e865c9 |
|
|
|
e865c9 |
|
|
|
e865c9 |
public:
|
|
|
e865c9 |
Trainer():
|
|
|
e865c9 |
barrierCounter(0),
|
|
|
e865c9 |
layer(),
|
|
|
e865c9 |
ratio(),
|
|
|
e865c9 |
threadsCount(1),
|
|
|
e865c9 |
itersPerBlock(100),
|
|
|
e865c9 |
blocksPerSaving(),
|
|
|
e865c9 |
blocksCount(1000),
|
|
|
e865c9 |
qmin(),
|
|
|
8e5348 |
skipBackpass(0),
|
|
|
e865c9 |
fl(),
|
|
|
e865c9 |
bl() { }
|
|
|
e865c9 |
|
|
|
e865c9 |
|
|
|
e865c9 |
Trainer& configure(
|
|
|
e865c9 |
Layer &layer,
|
|
|
e865c9 |
AccumReal ratio,
|
|
|
e865c9 |
int threadsCount,
|
|
|
e865c9 |
int itersPerBlock,
|
|
|
e865c9 |
int blocksPerSaving,
|
|
|
e865c9 |
int blocksCount,
|
|
|
e865c9 |
AccumReal qmin )
|
|
|
e865c9 |
{
|
|
|
e865c9 |
this->layer = &layer;
|
|
|
e865c9 |
this->ratio = ratio;
|
|
|
e865c9 |
this->threadsCount = threadsCount;
|
|
|
e865c9 |
this->itersPerBlock = itersPerBlock;
|
|
|
e865c9 |
this->blocksPerSaving = blocksPerSaving;
|
|
|
e865c9 |
this->blocksCount = blocksCount;
|
|
|
e865c9 |
this->qmin = qmin;
|
|
|
e865c9 |
return *this;
|
|
|
e865c9 |
}
|
|
|
e865c9 |
|
|
|
e865c9 |
|
|
|
8e5348 |
Quality run() {
|
|
|
e865c9 |
assert(layer && !layer->prev && layer->next);
|
|
|
e865c9 |
assert(threadsCount > 0);
|
|
|
e865c9 |
assert(itersPerBlock > 0);
|
|
|
e865c9 |
|
|
|
8e5348 |
Quality bad = {INFINITY, INFINITY};
|
|
|
8e5348 |
|
|
|
e865c9 |
printf("training: threads %d, itersPerBlock %d, ratio: %lf\n", threadsCount, itersPerBlock, ratio);
|
|
|
8e5348 |
fflush(stdout);
|
|
|
e865c9 |
|
|
|
e865c9 |
fl = layer;
|
|
|
e865c9 |
bl = &layer->back();
|
|
|
e865c9 |
|
|
|
e865c9 |
qualities.clear();
|
|
|
8e5348 |
qualities.resize(threadsCount, Quality{});
|
|
|
e865c9 |
for(Layer *l = layer; l; l = l->next)
|
|
|
e865c9 |
l->split(threadsCount);
|
|
|
e865c9 |
|
|
|
e865c9 |
if (!prepare())
|
|
|
8e5348 |
return printf("cannot prepare\n"), bad;
|
|
|
e865c9 |
|
|
|
8e5348 |
|
|
|
8e5348 |
|
|
|
8e5348 |
AccumReal ratioCopy = ratio;
|
|
|
8e5348 |
Quality result = bad, best = result, saved = result;
|
|
|
e865c9 |
long long fullTimeStartUs = timeUs();
|
|
|
8e5348 |
ratio = 0;
|
|
|
e865c9 |
int i = 0;
|
|
|
8e5348 |
int bps = blocksPerSaving > 0 ? blocksPerSaving : 1;
|
|
|
8e5348 |
int nextSave = i + bps;
|
|
|
e865c9 |
while(true) {
|
|
|
e865c9 |
if (!prepareBlock()) {
|
|
|
e865c9 |
printf("cannot prepare block\n");
|
|
|
8e5348 |
result = bad;
|
|
|
e865c9 |
break;
|
|
|
e865c9 |
};
|
|
|
e865c9 |
|
|
|
e865c9 |
long long runTimeUs = timeUs();
|
|
|
e865c9 |
result = runThreads(i);
|
|
|
e865c9 |
runTimeUs = timeUs() - runTimeUs;
|
|
|
e865c9 |
|
|
|
e865c9 |
finishBlock();
|
|
|
e865c9 |
|
|
|
e865c9 |
long long t = timeUs();
|
|
|
e865c9 |
long long fullTimeUs = t - fullTimeStartUs;
|
|
|
e865c9 |
fullTimeStartUs = t;
|
|
|
e865c9 |
++i;
|
|
|
e865c9 |
|
|
|
8e5348 |
if (i == 1) saved = result;
|
|
|
8e5348 |
bool good = result < best;
|
|
|
8e5348 |
bool done = (blocksCount > 0 && i >= blocksCount) || result.human <= qmin;
|
|
|
8e5348 |
bool saving = ratio > 0 && (i >= nextSave || done) && result < saved;
|
|
|
8e5348 |
if (good) best = result;
|
|
|
8e5348 |
|
|
|
8e5348 |
printf("%4d, total %7d, avg.result %f (%f), best %f (%f), time: %f / %f%s\n",
|
|
|
8e5348 |
i, i*itersPerBlock,
|
|
|
8e5348 |
result.human, result.train, best.human, best.train,
|
|
|
8e5348 |
runTimeUs*0.000001, fullTimeUs*0.000001,
|
|
|
8e5348 |
(saving ? ", saving" : "" ) );
|
|
|
8e5348 |
fflush(stdout);
|
|
|
8e5348 |
|
|
|
8e5348 |
if (saving) {
|
|
|
8e5348 |
if (!layer->save()) {
|
|
|
8e5348 |
printf("saving failed\n");
|
|
|
8e5348 |
result = bad;
|
|
|
8e5348 |
break;
|
|
|
8e5348 |
}
|
|
|
8e5348 |
saved = result;
|
|
|
8e5348 |
nextSave += bps;
|
|
|
e865c9 |
}
|
|
|
e865c9 |
|
|
|
e865c9 |
if (done) break;
|
|
|
8e5348 |
ratio = ratioCopy;
|
|
|
e865c9 |
}
|
|
|
8e5348 |
ratio = ratioCopy;
|
|
|
e865c9 |
|
|
|
e865c9 |
finish();
|
|
|
e865c9 |
|
|
|
e865c9 |
return result;
|
|
|
e865c9 |
}
|
|
|
e865c9 |
};
|
|
|
e865c9 |
|
|
|
e865c9 |
|
|
|
e865c9 |
#endif
|