ccdb: handle obscure text-based format

martinlindhe · martinlindhe · commit 724b4b5e9d63 · 2018-04-12T14:32:43.000+02:00
diff --git a/Makefile b/Makefile
@@ -0,0 +1,2 @@
+install:
+	go install -v ./...
diff --git a/ccdb.go b/ccdb.go
@@ -0,0 +1,65 @@
+package subtitles
+
+// an obscure (to me) text-based subtitle format, can have extension .cc or .txt
+// found in The Way We Live Now (2001) BBC TV 1.txt
+// STATUS: incomplete detection and support
+
+import (
+	"log"
+	"strings"
+	"time"
+)
+
+func looksLikeCCDBCapture(s string) bool {
+	return strings.Contains(s, "[SUBTITLE]")
+}
+
+// NewFromCCDBCapture parses a ccdb capture text into []Caption, assumes s is a clean utf8 string
+func NewFromCCDBCapture(s string) (res Subtitle, err error) {
+	rows := strings.Split(s, "\n")
+	seq := 1
+	caption := Caption{Seq: seq}
+	parseText := false
+	for rowNum, row := range rows {
+		if len(row) > 1 && row[0] == '[' {
+			continue
+		}
+		if parseText {
+			// log.Println("TEXT:", row)
+			if row == "\r" || row == "" {
+				parseText = false // XXX until text = empty
+			} else {
+				row = strings.TrimSpace(row)
+				if row != "" {
+					caption.Text = append(caption.Text, row)
+				}
+			}
+			if strings.Join(caption.Text, "") != "" {
+				res.Captions = append(res.Captions, caption)
+				seq++
+				caption = Caption{Seq: seq}
+			}
+		} else if !parseText {
+			if row == "" {
+				if rowNum != len(rows)-1 {
+					log.Println("NOTICE: ccdb seem to have reached end of valid stream at row", rowNum, "of", len(rows))
+				}
+				break
+			}
+			// log.Println("TIME:", row)
+			parts := strings.SplitN(row, ",", 2)
+			if len(parts) == 2 {
+				caption.Start, _ = parseCCDBTime(parts[0])
+				caption.End, _ = parseCCDBTime(parts[1])
+			} else {
+				log.Println("TIME seq", seq, ", input row", (rowNum + 1), "error:", row)
+			}
+			parseText = true
+		}
+	}
+	return
+}
+
+func parseCCDBTime(s string) (time.Time, error) {
+	return parseTime(s)
+}
diff --git a/ccdb_test.go b/ccdb_test.go
@@ -0,0 +1,51 @@
+package subtitles
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewFromCCDBCapture(t *testing.T) {
+
+	in := "[SUBTITLE]\r\n" +
+		"[COLF]&HFFFFFF,[STYLE]no,[SIZE]10,[FONT]Arial\r\n" +
+		"00:00:16.24,00:01:25.82\r\n" +
+		"Whoa.                           \r\n" +
+		"\r\n" +
+		"00:01:31.45,00:01:33.62\r\n" +
+		"Go on. Get out.                 \r\n" +
+		"\r\n" +
+		"00:01:33.62,00:01:33.65\r\n" +
+		"                                \r\n" + // should disappear in the parsed captions
+		"\r\n" +
+		"00:01:33.65,00:01:34.81\r\n" +
+		"Out!                            \r\n" +
+		"\r\n"
+
+	expected := Subtitle{[]Caption{{
+		1,
+		makeTime(0, 0, 16, 24),
+		makeTime(0, 1, 25, 82),
+		[]string{"Whoa."},
+	}, Caption{
+		2,
+		makeTime(0, 1, 31, 45),
+		makeTime(0, 1, 33, 62),
+		[]string{"Go on. Get out."},
+	}, Caption{
+		3,
+		makeTime(0, 1, 33, 65),
+		makeTime(0, 1, 34, 81),
+		[]string{"Out!"},
+	}}}
+
+	res, err := NewFromCCDBCapture(in)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, expected, res)
+}
+
+func TestParseCCDBTime(t *testing.T) {
+	t1, _ := parseCCDBTime("00:00:16.24")
+	assert.Equal(t, makeTime(0, 0, 16, 24), t1)
+}
diff --git a/cleaner_test.go b/cleaner_test.go
@@ -7,7 +7,6 @@ import (
 )
 
 func TestRemoveAds(t *testing.T) {
-
 	in := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
@@ -36,6 +35,5 @@ func TestRemoveAds(t *testing.T) {
 		makeTime(0, 1, 11, 005),
 		[]string{"No ninja!"},
 	}}}
-
 	assert.Equal(t, &expected, in.RemoveAds())
 }
diff --git a/cmd/subber/subber.go b/cmd/subber/subber.go
@@ -93,7 +93,7 @@ func cleanupSub(data []byte, filterName string, keepAds bool, sync int) (string,
 func action(inFileName string) error {
 
 	ext := path.Ext(inFileName)
-	if ext == ".srt" {
+	if subtitles.LooksLikeTextSubtitle(inFileName) {
 		if !*dontTouch {
 			parseAndWriteSubFile(inFileName, *filterName, *keepAds, *sync)
 		}
@@ -103,7 +103,6 @@ func action(inFileName string) error {
 	subFileName := inFileName[:len(inFileName)-len(ext)] + ".srt"
 
 	if fileExists(subFileName) {
-
 		verboseMessage("Subs found locally in", subFileName, ", skipping download")
 
 		if !*dontTouch {
diff --git a/encoding_test.go b/encoding_test.go
@@ -8,13 +8,11 @@ import (
 )
 
 func TestLooksLikeLatin1(t *testing.T) {
-
 	assert.Equal(t, true, looksLikeLatin1([]byte("hall\xe5")))
 	assert.Equal(t, false, looksLikeLatin1([]byte("hallå")))
 }
 
 func TestReadFileAsUTF8(t *testing.T) {
-
 	f, err := os.Open("README.md")
 	assert.Equal(t, nil, err)
 
diff --git a/filter_caps_test.go b/filter_caps_test.go
@@ -7,20 +7,17 @@ import (
 )
 
 func TestFilterCapitalization(t *testing.T) {
-
 	in := Subtitle{Captions: []Caption{{
 		Seq:   1,
 		Start: makeTime(0, 0, 4, 630),
 		End:   makeTime(0, 0, 6, 18),
 		Text:  []string{"GO NINJA!", "NINJA GO!"},
 	}}}
-
 	expected := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"Go ninja!", "Ninja go!"},
 	}}}
-
 	assert.Equal(t, &expected, in.filterCapitalization())
 }
diff --git a/filter_html_test.go b/filter_html_test.go
@@ -7,20 +7,17 @@ import (
 )
 
 func TestFilterHTML(t *testing.T) {
-
 	in := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"<b>GO NINJA!</b>", "NINJA&nbsp;GO!"},
 	}}}
-
 	expected := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"GO NINJA!", "NINJA GO!"},
 	}}}
-
 	assert.Equal(t, &expected, in.filterHTML())
 }
diff --git a/filter_ocr_test.go b/filter_ocr_test.go
@@ -7,96 +7,81 @@ import (
 )
 
 func TestFilterOCRLower(t *testing.T) {
-
 	in := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"s0mething good"},
 	}}}
-
 	expected := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"something good"},
 	}}}
-
 	assert.Equal(t, &expected, in.filterOCR())
 }
 
 func TestFilterOCRUpper(t *testing.T) {
-
 	in := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"S0METHING GOOD"},
 	}}}
-
 	expected := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"SOMETHING GOOD"},
 	}}}
-
 	assert.Equal(t, &expected, in.filterOCR())
 }
 
 func TestFilterOCRUcFirst(t *testing.T) {
-
 	in := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"S0mething good"},
 	}}}
-
 	expected := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"Something good"},
 	}}}
-
 	assert.Equal(t, &expected, in.filterOCR())
 }
 
 func TestFilterOCREnglish(t *testing.T) {
-
 	in := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"l've got a feeling"},
 	}}}
-
 	expected := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"i've got a feeling"},
 	}}}
-
 	assert.Equal(t, &expected, in.filterOCR())
 }
 
 func TestFilterOCRCapitalization(t *testing.T) {
-
 	in := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"GAsPs slowly"},
 	}}}
-
 	expected := Subtitle{[]Caption{{
 		1,
 		makeTime(0, 0, 4, 630),
 		makeTime(0, 0, 6, 18),
 		[]string{"GASPS slowly"},
 	}}}
-
 	assert.Equal(t, &expected, in.filterOCR())
 }
diff --git a/finder_thesubdb_test.go b/finder_thesubdb_test.go
@@ -11,7 +11,6 @@ import (
 )
 
 func TestDownloadFromTheSubDb(t *testing.T) {
-
 	fileName := createZeroedTempFile(1024 * 1024 * 4)
 	defer os.Remove(fileName)
 
@@ -44,7 +43,6 @@ func subDbConformTest(t *testing.T, fileName string, expectedHash string) {
 }
 
 func TestSubDbHashFromFile(t *testing.T) {
-
 	// NOTE for this to work, run "./hash-conformance-deps" to fetch needed files
 
 	// http://thesubdb.com/api/samples/dexter.mp4
diff --git a/parser.go b/parser.go
@@ -1,11 +1,17 @@
 package subtitles
 
-import "fmt"
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+)
 
 // Parse tries to parse a subtitle
 func Parse(b []byte) (Subtitle, error) {
 	s := ConvertToUTF8(b)
-	if looksLikeSSA(s) {
+	if looksLikeCCDBCapture(s) {
+		return NewFromCCDBCapture(s)
+	} else if looksLikeSSA(s) {
 		return NewFromSSA(s)
 	} else if looksLikeDCSub(s) {
 		return NewFromDCSub(s)
@@ -14,3 +20,13 @@ func Parse(b []byte) (Subtitle, error) {
 	}
 	return Subtitle{}, fmt.Errorf("parse: unrecognized subtitle type")
 }
+
+// LooksLikeTextSubtitle returns true i byte stream seems to be of a recognized format
+func LooksLikeTextSubtitle(filename string) bool {
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		log.Fatal(err)
+	}
+	s := ConvertToUTF8(data)
+	return looksLikeCCDBCapture(s) || looksLikeSSA(s) || looksLikeDCSub(s) || looksLikeSRT(s)
+}
diff --git a/srt_test.go b/srt_test.go
diff --git a/ssa_test.go b/ssa_test.go
diff --git a/testdata/ccdb/ccdb_captured_file.txt b/testdata/ccdb/ccdb_captured_file.txt

Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,6 @@ import (`
`7`	`7`	`)`
`8`	`8`
`9`	`9`	`func TestRemoveAds(t *testing.T) {`
`10`		`-`
`11`	`10`	`in := Subtitle{[]Caption{{`
`12`	`11`	`1,`
`13`	`12`	`makeTime(0, 0, 4, 630),`
`@@ -36,6 +35,5 @@ func TestRemoveAds(t *testing.T) {`
`36`	`35`	`makeTime(0, 1, 11, 005),`
`37`	`36`	`[]string{"No ninja!"},`
`38`	`37`	`}}}`
`39`		`-`
`40`	`38`	`assert.Equal(t, &expected, in.RemoveAds())`
`41`	`39`	`}`
Original file line number	Diff line number	Diff line change
`@@ -8,13 +8,11 @@ import (`
`8`	`8`	`)`
`9`	`9`
`10`	`10`	`func TestLooksLikeLatin1(t *testing.T) {`
`11`		`-`
`12`	`11`	`assert.Equal(t, true, looksLikeLatin1([]byte("hall\xe5")))`
`13`	`12`	`assert.Equal(t, false, looksLikeLatin1([]byte("hallå")))`
`14`	`13`	`}`
`15`	`14`
`16`	`15`	`func TestReadFileAsUTF8(t *testing.T) {`
`17`		`-`
`18`	`16`	`f, err := os.Open("README.md")`
`19`	`17`	`assert.Equal(t, nil, err)`
`20`	`18`
Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,6 @@ import (`
`11`	`11`	`)`
`12`	`12`
`13`	`13`	`func TestDownloadFromTheSubDb(t *testing.T) {`
`14`		`-`
`15`	`14`	`fileName := createZeroedTempFile(1024 * 1024 * 4)`
`16`	`15`	`defer os.Remove(fileName)`
`17`	`16`
`@@ -44,7 +43,6 @@ func subDbConformTest(t *testing.T, fileName string, expectedHash string) {`
`44`	`43`	`}`
`45`	`44`
`46`	`45`	`func TestSubDbHashFromFile(t *testing.T) {`
`47`		`-`
`48`	`46`	`// NOTE for this to work, run "./hash-conformance-deps" to fetch needed files`
`49`	`47`
`50`	`48`	`// http://thesubdb.com/api/samples/dexter.mp4`