1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
|
package main
import (
"database/sql"
// ...
_ "github.com/marcboeker/go-duckdb"
)
func main() {
// Connect to DuckDB
db, err := sql.Open("duckdb", "test.duckdb")
if err != nil {
log.Fatal(err)
}
defer db.Close()
db.Exec("DROP TABLE IF EXISTS phonebook")
db.Exec("DROP TABLE IF EXISTS unmatched_lines")
// Create a table
_, err = db.Exec("CREATE TABLE IF NOT EXISTS phonebook (name VARCHAR,phone VARCHAR,UNIQUE(name, phone))")
if err != nil {
log.Fatal(err)
}
_, err = db.Exec("CREATE TABLE IF NOT EXISTS unmatched_lines (line TEXT,UNIQUE(line))")
if err != nil {
log.Fatal(err)
}
// Function to process a text file
processFile := func(filePath string, wg *sync.WaitGroup, semaphore chan struct{}) {
defer wg.Done()
defer func() { <-semaphore }() // Release the semaphore
// ......
ProcessLines(db, lines)
}
var wg sync.WaitGroup
semaphore := make(chan struct{}, 100) // Limit to 100 goroutines
for _:= range(1000){
wg.Add(1)
semaphore <- struct{}{} // Acquire the semaphore
go processFile(path, &wg, semaphore)
}
wg.Wait()
}
// 多行插入
func ProcessLines(db *sql.DB, lines []string) {
// 开始一个事务
tx, err := db.Begin()
if err != nil {
log.Fatal(err)
}
// 准备插入语句
stmtUnmatched, err := tx.Prepare("INSERT INTO unmatched_lines (line) VALUES (?)")
if err != nil {
log.Fatal(err)
}
defer stmtUnmatched.Close()
stmtPhonebook, err := tx.Prepare("INSERT INTO phonebook (name, phone) VALUES (?, ?)")
if err != nil {
log.Fatal(err)
}
defer stmtPhonebook.Close()
_, err := stmtUnmatched.Exec(result[1])
if err != nil {
fmt.Printf("Error inserting into unmatched_lines: %v\n", err)
}
_, err := stmtPhonebook.Exec(result[0], result[1])
if err != nil {
fmt.Printf("Error inserting into phonebook: %v\n", err)
}
// 提交事务 :并非提交，仍然在wal中
if err := tx.Commit(); err != nil {
log.Fatal(err)
}
}
|