Skip to content

Commit 8e79297

Browse files
theraysmiththeraysmith
committed
Final part of endian improvement. Adds big-endian support to lstm and fixes issue 518
1 parent 6ac31dc commit 8e79297

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+299
-376
lines changed

‎ccmain/linerec.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ void Tesseract::TrainLineRecognizer(const STRING& input_imagename,
4949
DocumentData images(lstmf_name);
5050
if (applybox_page > 0) {
5151
// Load existing document for the previous pages.
52-
if (!images.LoadDocument(lstmf_name.string(), "eng", 0, 0, NULL)) {
52+
if (!images.LoadDocument(lstmf_name.string(), 0, 0, nullptr)) {
5353
tprintf("Failed to read training data from %s!\n", lstmf_name.string());
5454
return;
5555
}

‎ccmain/tessedit.cpp

+2-6
Original file line numberDiff line numberDiff line change
@@ -188,13 +188,9 @@ bool Tesseract::init_tesseract_lang_data(
188188
#ifndef ANDROID_BUILD
189189
if (tessedit_ocr_engine_mode == OEM_LSTM_ONLY ||
190190
tessedit_ocr_engine_mode == OEM_TESSERACT_LSTM_COMBINED) {
191-
if (mgr->swap()) {
192-
tprintf("Error: LSTM requested on big-endian hardware!!\n");
193-
tprintf("Big-endian not yet supported! Loading tesseract.\n");
194-
tessedit_ocr_engine_mode.set_value(OEM_TESSERACT_ONLY);
195-
} else if (mgr->GetComponent(TESSDATA_LSTM, &fp)) {
191+
if (mgr->GetComponent(TESSDATA_LSTM, &fp)) {
196192
lstm_recognizer_ = new LSTMRecognizer;
197-
ASSERT_HOST(lstm_recognizer_->DeSerialize(mgr->swap(), &fp));
193+
ASSERT_HOST(lstm_recognizer_->DeSerialize(&fp));
198194
if (lstm_use_matrix) lstm_recognizer_->LoadDictionary(language, mgr);
199195
} else {
200196
tprintf("Error: LSTM requested, but not present!! Loading tesseract.\n");

‎ccstruct/fontinfo.cpp

+18-21
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@ bool FontInfo::Serialize(FILE* fp) const {
3131
}
3232
// Reads from the given file. Returns false in case of error.
3333
// If swap is true, assumes a big/little-endian swap is needed.
34-
bool FontInfo::DeSerialize(bool swap, TFile* fp) {
35-
if (!read_info(fp, this, swap)) return false;
36-
if (!read_spacing_info(fp, this, swap)) return false;
34+
bool FontInfo::DeSerialize(TFile* fp) {
35+
if (!read_info(fp, this)) return false;
36+
if (!read_spacing_info(fp, this)) return false;
3737
return true;
3838
}
3939

@@ -51,9 +51,9 @@ bool FontInfoTable::Serialize(FILE* fp) const {
5151
}
5252
// Reads from the given file. Returns false in case of error.
5353
// If swap is true, assumes a big/little-endian swap is needed.
54-
bool FontInfoTable::DeSerialize(bool swap, TFile* fp) {
54+
bool FontInfoTable::DeSerialize(TFile* fp) {
5555
truncate(0);
56-
return this->DeSerializeClasses(swap, fp);
56+
return this->DeSerializeClasses(fp);
5757
}
5858

5959
// Returns true if the given set of fonts includes one with the same
@@ -149,14 +149,14 @@ void FontSetDeleteCallback(FontSet fs) {
149149

150150
/*---------------------------------------------------------------------------*/
151151
// Callbacks used by UnicityTable to read/write FontInfo/FontSet structures.
152-
bool read_info(TFile* f, FontInfo* fi, bool swap) {
152+
bool read_info(TFile* f, FontInfo* fi) {
153153
inT32 size;
154-
if (f->FReadEndian(&size, sizeof(size), 1, swap) != 1) return false;
154+
if (f->FReadEndian(&size, sizeof(size), 1) != 1) return false;
155155
char* font_name = new char[size + 1];
156156
fi->name = font_name;
157157
if (f->FRead(font_name, sizeof(*font_name), size) != size) return false;
158158
font_name[size] = '\0';
159-
if (f->FReadEndian(&fi->properties, sizeof(fi->properties), 1, swap) != 1)
159+
if (f->FReadEndian(&fi->properties, sizeof(fi->properties), 1) != 1)
160160
return false;
161161
return true;
162162
}
@@ -170,28 +170,26 @@ bool write_info(FILE* f, const FontInfo& fi) {
170170
return true;
171171
}
172172

173-
bool read_spacing_info(TFile* f, FontInfo* fi, bool swap) {
173+
bool read_spacing_info(TFile* f, FontInfo* fi) {
174174
inT32 vec_size, kern_size;
175-
if (f->FReadEndian(&vec_size, sizeof(vec_size), 1, swap) != 1) return false;
175+
if (f->FReadEndian(&vec_size, sizeof(vec_size), 1) != 1) return false;
176176
ASSERT_HOST(vec_size >= 0);
177177
if (vec_size == 0) return true;
178178
fi->init_spacing(vec_size);
179179
for (int i = 0; i < vec_size; ++i) {
180180
FontSpacingInfo *fs = new FontSpacingInfo();
181-
if (f->FReadEndian(&fs->x_gap_before, sizeof(fs->x_gap_before), 1, swap) !=
182-
1 ||
183-
f->FReadEndian(&fs->x_gap_after, sizeof(fs->x_gap_after), 1, swap) !=
184-
1 ||
185-
f->FReadEndian(&kern_size, sizeof(kern_size), 1, swap) != 1) {
181+
if (f->FReadEndian(&fs->x_gap_before, sizeof(fs->x_gap_before), 1) != 1 ||
182+
f->FReadEndian(&fs->x_gap_after, sizeof(fs->x_gap_after), 1) != 1 ||
183+
f->FReadEndian(&kern_size, sizeof(kern_size), 1) != 1) {
186184
delete fs;
187185
return false;
188186
}
189187
if (kern_size < 0) { // indication of a NULL entry in fi->spacing_vec
190188
delete fs;
191189
continue;
192190
}
193-
if (kern_size > 0 && (!fs->kerned_unichar_ids.DeSerialize(swap, f) ||
194-
!fs->kerned_x_gaps.DeSerialize(swap, f))) {
191+
if (kern_size > 0 && (!fs->kerned_unichar_ids.DeSerialize(f) ||
192+
!fs->kerned_x_gaps.DeSerialize(f))) {
195193
delete fs;
196194
return false;
197195
}
@@ -229,11 +227,10 @@ bool write_spacing_info(FILE* f, const FontInfo& fi) {
229227
return true;
230228
}
231229

232-
bool read_set(TFile* f, FontSet* fs, bool swap) {
233-
if (f->FReadEndian(&fs->size, sizeof(fs->size), 1, swap) != 1) return false;
230+
bool read_set(TFile* f, FontSet* fs) {
231+
if (f->FReadEndian(&fs->size, sizeof(fs->size), 1) != 1) return false;
234232
fs->configs = new int[fs->size];
235-
if (f->FReadEndian(fs->configs, sizeof(fs->configs[0]), fs->size, swap) !=
236-
fs->size)
233+
if (f->FReadEndian(fs->configs, sizeof(fs->configs[0]), fs->size) != fs->size)
237234
return false;
238235
return true;
239236
}

‎ccstruct/fontinfo.h

+5-5
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ struct FontInfo {
6767
bool Serialize(FILE* fp) const;
6868
// Reads from the given file. Returns false in case of error.
6969
// If swap is true, assumes a big/little-endian swap is needed.
70-
bool DeSerialize(bool swap, TFile* fp);
70+
bool DeSerialize(TFile* fp);
7171

7272
// Reserves unicharset_size spots in spacing_vec.
7373
void init_spacing(int unicharset_size) {
@@ -152,7 +152,7 @@ class FontInfoTable : public GenericVector<FontInfo> {
152152
bool Serialize(FILE* fp) const;
153153
// Reads from the given file. Returns false in case of error.
154154
// If swap is true, assumes a big/little-endian swap is needed.
155-
bool DeSerialize(bool swap, TFile* fp);
155+
bool DeSerialize(TFile* fp);
156156

157157
// Returns true if the given set of fonts includes one with the same
158158
// properties as font_id.
@@ -177,11 +177,11 @@ void FontInfoDeleteCallback(FontInfo f);
177177
void FontSetDeleteCallback(FontSet fs);
178178

179179
// Callbacks used by UnicityTable to read/write FontInfo/FontSet structures.
180-
bool read_info(TFile* f, FontInfo* fi, bool swap);
180+
bool read_info(TFile* f, FontInfo* fi);
181181
bool write_info(FILE* f, const FontInfo& fi);
182-
bool read_spacing_info(TFile* f, FontInfo* fi, bool swap);
182+
bool read_spacing_info(TFile* f, FontInfo* fi);
183183
bool write_spacing_info(FILE* f, const FontInfo& fi);
184-
bool read_set(TFile* f, FontSet* fs, bool swap);
184+
bool read_set(TFile* f, FontSet* fs);
185185
bool write_set(FILE* f, const FontSet& fs);
186186

187187
} // namespace tesseract.

‎ccstruct/imagedata.cpp

+29-27
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ bool ImageData::Serialize(TFile* fp) const {
166166
if (!imagefilename_.Serialize(fp)) return false;
167167
if (fp->FWrite(&page_number_, sizeof(page_number_), 1) != 1) return false;
168168
if (!image_data_.Serialize(fp)) return false;
169+
if (!language_.Serialize(fp)) return false;
169170
if (!transcription_.Serialize(fp)) return false;
170171
// WARNING: Will not work across different endian machines.
171172
if (!boxes_.Serialize(fp)) return false;
@@ -177,30 +178,32 @@ bool ImageData::Serialize(TFile* fp) const {
177178

178179
// Reads from the given file. Returns false in case of error.
179180
// If swap is true, assumes a big/little-endian swap is needed.
180-
bool ImageData::DeSerialize(bool swap, TFile* fp) {
181-
if (!imagefilename_.DeSerialize(swap, fp)) return false;
182-
if (fp->FRead(&page_number_, sizeof(page_number_), 1) != 1) return false;
183-
if (swap) ReverseN(&page_number_, sizeof(page_number_));
184-
if (!image_data_.DeSerialize(swap, fp)) return false;
185-
if (!transcription_.DeSerialize(swap, fp)) return false;
181+
bool ImageData::DeSerialize(TFile* fp) {
182+
if (!imagefilename_.DeSerialize(fp)) return false;
183+
if (fp->FReadEndian(&page_number_, sizeof(page_number_), 1) != 1)
184+
return false;
185+
if (!image_data_.DeSerialize(fp)) return false;
186+
if (!language_.DeSerialize(fp)) return false;
187+
if (!transcription_.DeSerialize(fp)) return false;
186188
// WARNING: Will not work across different endian machines.
187-
if (!boxes_.DeSerialize(swap, fp)) return false;
188-
if (!box_texts_.DeSerializeClasses(swap, fp)) return false;
189+
if (!boxes_.DeSerialize(fp)) return false;
190+
if (!box_texts_.DeSerializeClasses(fp)) return false;
189191
inT8 vertical = 0;
190192
if (fp->FRead(&vertical, sizeof(vertical), 1) != 1) return false;
191193
vertical_text_ = vertical != 0;
192194
return true;
193195
}
194196

195197
// As DeSerialize, but only seeks past the data - hence a static method.
196-
bool ImageData::SkipDeSerialize(bool swap, TFile* fp) {
197-
if (!STRING::SkipDeSerialize(swap, fp)) return false;
198+
bool ImageData::SkipDeSerialize(TFile* fp) {
199+
if (!STRING::SkipDeSerialize(fp)) return false;
198200
inT32 page_number;
199201
if (fp->FRead(&page_number, sizeof(page_number), 1) != 1) return false;
200-
if (!GenericVector<char>::SkipDeSerialize(swap, fp)) return false;
201-
if (!STRING::SkipDeSerialize(swap, fp)) return false;
202-
if (!GenericVector<TBOX>::SkipDeSerialize(swap, fp)) return false;
203-
if (!GenericVector<STRING>::SkipDeSerializeClasses(swap, fp)) return false;
202+
if (!GenericVector<char>::SkipDeSerialize(fp)) return false;
203+
if (!STRING::SkipDeSerialize(fp)) return false;
204+
if (!STRING::SkipDeSerialize(fp)) return false;
205+
if (!GenericVector<TBOX>::SkipDeSerialize(fp)) return false;
206+
if (!GenericVector<STRING>::SkipDeSerializeClasses(fp)) return false;
204207
inT8 vertical = 0;
205208
return fp->FRead(&vertical, sizeof(vertical), 1) == 1;
206209
}
@@ -384,21 +387,19 @@ DocumentData::~DocumentData() {
384387

385388
// Reads all the pages in the given lstmf filename to the cache. The reader
386389
// is used to read the file.
387-
bool DocumentData::LoadDocument(const char* filename, const char* lang,
388-
int start_page, inT64 max_memory,
389-
FileReader reader) {
390-
SetDocument(filename, lang, max_memory, reader);
390+
bool DocumentData::LoadDocument(const char* filename, int start_page,
391+
inT64 max_memory, FileReader reader) {
392+
SetDocument(filename, max_memory, reader);
391393
pages_offset_ = start_page;
392394
return ReCachePages();
393395
}
394396

395397
// Sets up the document, without actually loading it.
396-
void DocumentData::SetDocument(const char* filename, const char* lang,
397-
inT64 max_memory, FileReader reader) {
398+
void DocumentData::SetDocument(const char* filename, inT64 max_memory,
399+
FileReader reader) {
398400
SVAutoLock lock_p(&pages_mutex_);
399401
SVAutoLock lock(&general_mutex_);
400402
document_name_ = filename;
401-
lang_ = lang;
402403
pages_offset_ = -1;
403404
max_memory_ = max_memory;
404405
reader_ = reader;
@@ -522,7 +523,7 @@ bool DocumentData::ReCachePages() {
522523
pages_.truncate(0);
523524
TFile fp;
524525
if (!fp.Open(document_name_, reader_) ||
525-
!PointerVector<ImageData>::DeSerializeSize(false, &fp, &loaded_pages) ||
526+
!PointerVector<ImageData>::DeSerializeSize(&fp, &loaded_pages) ||
526527
loaded_pages <= 0) {
527528
tprintf("Deserialize header failed: %s\n", document_name_.string());
528529
return false;
@@ -534,15 +535,17 @@ bool DocumentData::ReCachePages() {
534535
for (page = 0; page < loaded_pages; ++page) {
535536
if (page < pages_offset_ ||
536537
(max_memory_ > 0 && memory_used() > max_memory_)) {
537-
if (!PointerVector<ImageData>::DeSerializeSkip(false, &fp)) break;
538+
if (!PointerVector<ImageData>::DeSerializeSkip(&fp)) {
539+
tprintf("Deserializeskip failed\n");
540+
break;
541+
}
538542
} else {
539-
if (!pages_.DeSerializeElement(false, &fp)) break;
543+
if (!pages_.DeSerializeElement(&fp)) break;
540544
ImageData* image_data = pages_.back();
541545
if (image_data->imagefilename().length() == 0) {
542546
image_data->set_imagefilename(document_name_);
543547
image_data->set_page_number(page);
544548
}
545-
image_data->set_language(lang_);
546549
set_memory_used(memory_used() + image_data->MemoryUsed());
547550
}
548551
}
@@ -567,7 +570,6 @@ DocumentCache::~DocumentCache() {}
567570
// Adds all the documents in the list of filenames, counting memory.
568571
// The reader is used to read the files.
569572
bool DocumentCache::LoadDocuments(const GenericVector<STRING>& filenames,
570-
const char* lang,
571573
CachingStrategy cache_strategy,
572574
FileReader reader) {
573575
cache_strategy_ = cache_strategy;
@@ -580,7 +582,7 @@ bool DocumentCache::LoadDocuments(const GenericVector<STRING>& filenames,
580582
for (int arg = 0; arg < filenames.size(); ++arg) {
581583
STRING filename = filenames[arg];
582584
DocumentData* document = new DocumentData(filename);
583-
document->SetDocument(filename.string(), lang, fair_share_memory, reader);
585+
document->SetDocument(filename.string(), fair_share_memory, reader);
584586
AddToCache(document);
585587
}
586588
if (!documents_.empty()) {

‎ccstruct/imagedata.h

+6-10
Original file line numberDiff line numberDiff line change
@@ -116,10 +116,9 @@ class ImageData {
116116
// Writes to the given file. Returns false in case of error.
117117
bool Serialize(TFile* fp) const;
118118
// Reads from the given file. Returns false in case of error.
119-
// If swap is true, assumes a big/little-endian swap is needed.
120-
bool DeSerialize(bool swap, TFile* fp);
119+
bool DeSerialize(TFile* fp);
121120
// As DeSerialize, but only seeks past the data - hence a static method.
122-
static bool SkipDeSerialize(bool swap, tesseract::TFile* fp);
121+
static bool SkipDeSerialize(tesseract::TFile* fp);
123122

124123
// Other accessors.
125124
const STRING& imagefilename() const {
@@ -210,11 +209,10 @@ class DocumentData {
210209

211210
// Reads all the pages in the given lstmf filename to the cache. The reader
212211
// is used to read the file.
213-
bool LoadDocument(const char* filename, const char* lang, int start_page,
214-
inT64 max_memory, FileReader reader);
212+
bool LoadDocument(const char* filename, int start_page, inT64 max_memory,
213+
FileReader reader);
215214
// Sets up the document, without actually loading it.
216-
void SetDocument(const char* filename, const char* lang, inT64 max_memory,
217-
FileReader reader);
215+
void SetDocument(const char* filename, inT64 max_memory, FileReader reader);
218216
// Writes all the pages to the given filename. Returns false on error.
219217
bool SaveDocument(const char* filename, FileWriter writer);
220218
bool SaveToBuffer(GenericVector<char>* buffer);
@@ -286,8 +284,6 @@ class DocumentData {
286284
private:
287285
// A name for this document.
288286
STRING document_name_;
289-
// The language of this document.
290-
STRING lang_;
291287
// A group of pages that corresponds in some loose way to a document.
292288
PointerVector<ImageData> pages_;
293289
// Page number of the first index in pages_.
@@ -325,7 +321,7 @@ class DocumentCache {
325321
}
326322
// Adds all the documents in the list of filenames, counting memory.
327323
// The reader is used to read the files.
328-
bool LoadDocuments(const GenericVector<STRING>& filenames, const char* lang,
324+
bool LoadDocuments(const GenericVector<STRING>& filenames,
329325
CachingStrategy cache_strategy, FileReader reader);
330326

331327
// Adds document to the cache.

‎ccstruct/matrix.h

+7-16
Original file line numberDiff line numberDiff line change
@@ -164,16 +164,11 @@ class GENERIC_2D_ARRAY {
164164
}
165165
return true;
166166
}
167-
bool DeSerialize(bool swap, tesseract::TFile* fp) {
168-
if (!DeSerializeSize(swap, fp)) return false;
169-
if (fp->FRead(&empty_, sizeof(empty_), 1) != 1) return false;
170-
if (swap) ReverseN(&empty_, sizeof(empty_));
167+
bool DeSerialize(tesseract::TFile* fp) {
168+
if (!DeSerializeSize(fp)) return false;
169+
if (fp->FReadEndian(&empty_, sizeof(empty_), 1) != 1) return false;
171170
int size = num_elements();
172-
if (fp->FRead(array_, sizeof(*array_), size) != size) return false;
173-
if (swap) {
174-
for (int i = 0; i < size; ++i)
175-
ReverseN(&array_[i], sizeof(array_[i]));
176-
}
171+
if (fp->FReadEndian(array_, sizeof(*array_), size) != size) return false;
177172
return true;
178173
}
179174

@@ -487,14 +482,10 @@ class GENERIC_2D_ARRAY {
487482
Resize(size1, size2, empty_);
488483
return true;
489484
}
490-
bool DeSerializeSize(bool swap, tesseract::TFile* fp) {
485+
bool DeSerializeSize(tesseract::TFile* fp) {
491486
inT32 size1, size2;
492-
if (fp->FRead(&size1, sizeof(size1), 1) != 1) return false;
493-
if (fp->FRead(&size2, sizeof(size2), 1) != 1) return false;
494-
if (swap) {
495-
ReverseN(&size1, sizeof(size1));
496-
ReverseN(&size2, sizeof(size2));
497-
}
487+
if (fp->FReadEndian(&size1, sizeof(size1), 1) != 1) return false;
488+
if (fp->FReadEndian(&size2, sizeof(size2), 1) != 1) return false;
498489
Resize(size1, size2, empty_);
499490
return true;
500491
}

0 commit comments

Comments
 (0)