diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 81260901e044ebe952cedacfebfba6f0447142eb..f77265b9de71e5beef1e4d87591f6f4613aa82c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,32 +4,30 @@ stages: variables: GIT_SUBMODULE_STRATEGY: recursive -gt_checkout_build_x86: +gt-checksum_build_x86: stage: build retry: 2 only: - - schedules - - master + - v1.2.2 tags: - - gt-tools-builder-x86 + - golangci-x86 script: - - /bin/bash build-x86.sh + - /bin/bash build.sh artifacts: paths: - - binary/ - expire_in: 30 days + - release/ + expire_in: 7 days -gt_checkout_build_arm: +gt-checksum_build_arm: stage: build retry: 2 only: - - schedules - - master + - v1.2.2 tags: - - gt-tools-builder-arm + - golangci-arm script: - - /bin/bash build-arm.sh + - /bin/bash build.sh artifacts: paths: - - binary/ - expire_in: 30 days \ No newline at end of file + - release/ + expire_in: 7 days diff --git a/relnotes/CHANGELOG.zh-CN.md b/CHANGELOG.md similarity index 59% rename from relnotes/CHANGELOG.zh-CN.md rename to CHANGELOG.md index e97b4fe109675ea4b14ccccd6df6590dcc17e651..857dec3fa41d79f5b6a6e6c7b624cd8d2d722cc6 100644 --- a/relnotes/CHANGELOG.zh-CN.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +## 1.2.2 +- 合并`jointIndexChanRowCount`和`singleIndexChanRowCount`两个参数为新的参数`chunkSize` +- 不再支持命令行传参方式调用，仅支持配置文件方式调用，命令行参数仅支持"-h", "-v", "-c"等几个必要的参数 +- 删除极简模式，默认支持配置文件中只有srcDSN, dstDSN, tables等几个参数 +- 参数名`lowerCaseTableNames`变更为`caseSensitiveObjectName`，更好理解 +- 新增参数`memoryLimit`，用于限制内存使用量，防止OOM +- 优化校验结果输出，Rows的值改为精确值，此外不再频繁输出刷屏 +- 参数`logFile`支持日期时间格式，例如：gt-checksum-%Y%m%d%H%M%S.log +- 优化校验结果进度条及汇总报告内容，增加各表、各阶段各自的耗时 +- 修复无法使用普通索引和无索引时校验失败的问题 +- Bugs fixed + - 命令行 ScheckFixRule 参数传入失败问题 #IA84QZ + - 检查不出来数据不一致问题 #I8HSQB + - 空表直接报错以及表名大小问题 #I8SEPI + - out of memory问题 #I89A2J + - 校验输出结果中Rows数值不精确问题 #I830CY + - 表统计信息为空导致运行失败问题 #I7Y64J + ## 1.2.1 新增表结构校验、列类型校验等新特性及修复数个bug。 `gt-checksum` 修复bug及新增功能如下： @@ -5,9 +23,9 @@ - 支持列的数据类型的校验及修复 - 支持列的字符集及校验级的校验及修复（MySQL支持字符串校验，oracle不校验） - 支持列是否允许null的校验及修复 -- 支持列的默认值是否一致的校验及修复 +- 支持列的默认值是否一致的校验及修复 - 支持列的乱序的验证及修复 -- 支持列数据存在多列、少列的验证及修复 +- 支持列数据存在多列、少列的验证及修复 - 支持列的comment的校验及修复 - 支持宽松模式和严谨模式校验 - 支持校验列时是按正序校验还是乱序校验 diff --git a/Dockerfile b/Dockerfile index 3c1cae692f26bd3b9bb48a020a4e6e01552b9c3c..7c1270f27fad053f371aa418c9a27bfa9304efc2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,9 @@ # VERSION 1.2.1 # Author: greatsql # Command format: Instruction [arguments / command] … +# 本Dockerfile适用于Docker 17.05及以上版本，如果你的Docker版本较低，请先升级你的Docker +# 如果是podman则最后可能无法正常运行，因为podman不支持-o选项，可以改用buildah(4.x以上)实现，例如 +# DOCKER_BUILDKIT=1 buildah build-using-dockerfile -t greatsql/gt-checksum:1.2.1 -f Dockerfile . FROM golang:latest AS builder @@ -21,8 +24,8 @@ COPY . . ARG VERSION RUN go mod tidy -RUN go build -o gt-checksum greatdbCheck.go -RUN mkdir -p ./gt-checksum-${VERSION} && cp -rf docs gc.conf gc.conf-simple gt-checksum Oracle/instantclient_11_2 README.md relnotes gt-checksum-${VERSION} +RUN go build -o gt-checksum gt-checksum.go +RUN mkdir -p ./gt-checksum-${VERSION} && cp -rf README.md CHANGELOG.zh-CN.md gc-sample.conf gt-checksum Oracle/instantclient_11_2 gt-checksum-${VERSION} FROM scratch AS exporter diff --git a/MySQL/binlogEventInfo.go b/MySQL/binlogEventInfo.go index 7c3140e7e4e5731dc48f822d81eea98eba8a84d0..4d2a3572f077a3632c58351d3ad8bfd14e5ef252 100644 --- a/MySQL/binlogEventInfo.go +++ b/MySQL/binlogEventInfo.go @@ -53,13 +53,13 @@ func (my MySQLIncDataBinlogPrepareStruct) binlogSqlTransition(a []BinlogPrepareS table := strings.Split(a[i].dmlTableName, "/*SchemaTable*/")[1] switch a[i].dmlSqlType { case "insert": - tmpSql := fmt.Sprintf("insert into `%s`.`%s` values (%s);", schema, table, strings.Join(a[i].dmlSqlCollection, "),(")) + tmpSql := fmt.Sprintf("INSERT INTO `%s`.`%s` VALUES (%s);", schema, table, strings.Join(a[i].dmlSqlCollection, "),(")) sql = append(sql, tmpSql) case "update": - tmpSql := fmt.Sprintf("update `%s`.`%s` where (%s);", schema, table, strings.Join(a[i].dmlSqlCollection, "),(")) + tmpSql := fmt.Sprintf("UPDATE `%s`.`%s` WHERE (%s);", schema, table, strings.Join(a[i].dmlSqlCollection, "),(")) sql = append(sql, tmpSql) case "delete": - tmpSql := fmt.Sprintf("delete from `%s`.`%s` where (%s);", schema, table, strings.Join(a[i].dmlSqlCollection, "),(")) + tmpSql := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE (%s);", schema, table, strings.Join(a[i].dmlSqlCollection, "),(")) sql = append(sql, tmpSql) } } diff --git a/MySQL/my_data_fix_sql.go b/MySQL/my_data_fix_sql.go index 86f0b5641e3b57d3a2c26170c3957ec6799ed7f7..a0d077c7ae1f509ad4c3105a060f5bd5dad945ea 100644 --- a/MySQL/my_data_fix_sql.go +++ b/MySQL/my_data_fix_sql.go @@ -72,7 +72,7 @@ func (my *MysqlDataAbnormalFixStruct) FixInsertSqlExec(db *sql.DB, sourceDrive s //} if len(valuesNameSeq) > 0 { queryColumn := strings.Join(valuesNameSeq, ",") - insertSql = fmt.Sprintf("insert into `%s`.`%s` values(%s) ;", my.Schema, my.Table, queryColumn) + insertSql = fmt.Sprintf("INSERT INTO `%s`.`%s` VALUES(%s) ;", my.Schema, my.Table, queryColumn) } //if strings.Contains(queryColumn, "''") { // insertSql = fmt.Sprintf("insert into `%s`.`%s` values(%s) ;", my.Schema, my.Table, strings.ReplaceAll(queryColumn, "''", "NULL")) @@ -107,7 +107,7 @@ func (my *MysqlDataAbnormalFixStruct) FixDeleteSqlExec(db *sql.DB, sourceDrive s global.Wlog.Debug(vlog) vlog = fmt.Sprintf("(%d) MySQL DB check table %s.%s Generate delete repair statement based on unique index.", logThreadSeq, my.Schema, my.Table) global.Wlog.Debug(vlog) - if my.IndexType == "mui" { + if my.IndexType == "mul" { var FB, AS []string for _, i := range colData { FB = append(FB, i["columnName"]) @@ -115,11 +115,11 @@ func (my *MysqlDataAbnormalFixStruct) FixDeleteSqlExec(db *sql.DB, sourceDrive s rowData := strings.ReplaceAll(my.RowData, "/*go actions columnData*//*go actions columnData*/", "/*go actions columnData*/greatdbNull/*go actions columnData*/") for k, v := range strings.Split(rowData, "/*go actions columnData*/") { if v == "" { - AS = append(AS, fmt.Sprintf(" %s is null ", FB[k])) + AS = append(AS, fmt.Sprintf(" %s IS NULL ", FB[k])) } else if v == "" { AS = append(AS, fmt.Sprintf(" %s = ''", FB[k])) } else if v == acc["double"] { - AS = append(AS, fmt.Sprintf(" concat(%s,'') = '%s'", FB[k], v)) + AS = append(AS, fmt.Sprintf(" CONCAT(%s,'') = '%s'", FB[k], v)) } else { AS = append(AS, fmt.Sprintf(" %s = '%s' ", FB[k], v)) } @@ -140,21 +140,21 @@ func (my *MysqlDataAbnormalFixStruct) FixDeleteSqlExec(db *sql.DB, sourceDrive s for l, I := range FB { if I == strconv.Itoa(k+1) { if v == "" { - AS = append(AS, fmt.Sprintf(" %s is null ", my.IndexColumn[l])) + AS = append(AS, fmt.Sprintf(" %s IS NULL ", my.IndexColumn[l])) } else if v == "" { AS = append(AS, fmt.Sprintf(" %s = '' ", my.IndexColumn[l])) } else if v == acc["double"] { - AS = append(AS, fmt.Sprintf(" concat(%s,'') = '%s'", my.IndexColumn[l], v)) + AS = append(AS, fmt.Sprintf(" CONCAT(%s,'') = '%s'", my.IndexColumn[l], v)) } else { AS = append(AS, fmt.Sprintf(" %s = '%s' ", my.IndexColumn[l], v)) } } - deleteSqlWhere = strings.Join(AS, " and ") + deleteSqlWhere = strings.Join(AS, " AND ") } } } if len(deleteSqlWhere) > 0 { - deleteSql = fmt.Sprintf("delete from `%s`.`%s` where %s;", my.Schema, my.Table, deleteSqlWhere) + deleteSql = fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE %s;", my.Schema, my.Table, deleteSqlWhere) } return deleteSql, nil } @@ -169,22 +169,22 @@ func (my *MysqlDataAbnormalFixStruct) FixAlterIndexSqlExec(e, f []string, si map } switch my.IndexType { case "pri": - strsql = fmt.Sprintf("alter table `%s`.`%s` add primary key(`%s`);", my.Schema, my.Table, strings.Join(c, "`,`")) + strsql = fmt.Sprintf("ALTER TABLE `%s`.`%s` ADD PRIMARY KEY(`%s`);", my.Schema, my.Table, strings.Join(c, "`,`")) case "uni": - strsql = fmt.Sprintf("alter table `%s`.`%s` add unique index %s(`%s`);", my.Schema, my.Table, v, strings.Join(c, "`,`")) + strsql = fmt.Sprintf("ALTER TABLE `%s`.`%s` ADD UNIQUE INDEX %s(`%s`);", my.Schema, my.Table, v, strings.Join(c, "`,`")) case "mul": - strsql = fmt.Sprintf("alter table `%s`.`%s` add index %s(`%s`);", my.Schema, my.Table, v, strings.Join(c, "`,`")) + strsql = fmt.Sprintf("ALTER TABLE `%s`.`%s` ADD INDEX %s(`%s`);", my.Schema, my.Table, v, strings.Join(c, "`,`")) } sqlS = append(sqlS, strsql) } for _, v := range f { switch my.IndexType { case "pri": - strsql = fmt.Sprintf("alter table `%s`.`%s` drop primary key;", my.Schema, my.Table) + strsql = fmt.Sprintf("ALTER TABLE `%s`.`%s` DROP PRIMARY KEY;", my.Schema, my.Table) case "uni": - strsql = fmt.Sprintf("alter table `%s.`%s drop index %s;", my.Schema, my.Table, v) + strsql = fmt.Sprintf("ALTER TABLE `%s.`%s DROP INDEX %s;", my.Schema, my.Table, v) case "mul": - strsql = fmt.Sprintf("alter table `%s`.`%s` drop index %s;", my.Schema, my.Table, v) + strsql = fmt.Sprintf("ALTER TABLE `%s`.`%s` DROP INDEX %s;", my.Schema, my.Table, v) } sqlS = append(sqlS, strsql) } @@ -195,51 +195,51 @@ func (my *MysqlDataAbnormalFixStruct) FixAlterColumnSqlDispos(alterType string, var sqlS string charsetN := "" if columnDataType[1] != "null" { - charsetN = fmt.Sprintf("character set %s", columnDataType[1]) + charsetN = fmt.Sprintf("CHARACTER SET %s", columnDataType[1]) } collationN := "" if columnDataType[2] != "null" { - collationN = fmt.Sprintf("collate %s", columnDataType[2]) + collationN = fmt.Sprintf("COLLATE %s", columnDataType[2]) } nullS := "" if strings.ToUpper(columnDataType[3]) == "NO" { - nullS = "not null" + nullS = "NOT NULL" } collumnDefaultN := "" if columnDataType[4] == "empty" { - collumnDefaultN = fmt.Sprintf("default ''") - } else if columnDataType[4] == "null" { + collumnDefaultN = fmt.Sprintf("DEFAULT ''") + } else if columnDataType[4] == "NULL" { collumnDefaultN = "" } else { - collumnDefaultN = fmt.Sprintf("default '%s'", columnDataType[4]) + collumnDefaultN = fmt.Sprintf("DEFAULT '%s'", columnDataType[4]) } commantS := "" if columnDataType[5] != "empty" { - commantS = fmt.Sprintf("comment '%s'", columnDataType[5]) + commantS = fmt.Sprintf("COMMENT '%s'", columnDataType[5]) } columnLocation := "" if columnSeq == 0 { - columnLocation = "first" + columnLocation = "FIRST" } else { if lastColumn != "alterNoAfter" { - columnLocation = fmt.Sprintf("after `%s`", lastColumn) + columnLocation = fmt.Sprintf("AFTER `%s`", lastColumn) } } switch alterType { case "add": - sqlS = fmt.Sprintf(" add column `%s` %s %s %s %s %s %s %s", curryColumn, columnDataType[0], charsetN, collationN, nullS, collumnDefaultN, commantS, columnLocation) + sqlS = fmt.Sprintf(" ADD COLUMN `%s` %s %s %s %s %s %s %s", curryColumn, columnDataType[0], charsetN, collationN, nullS, collumnDefaultN, commantS, columnLocation) case "modify": - sqlS = fmt.Sprintf(" modify column `%s` %s %s %s %s %s %s %s", curryColumn, columnDataType[0], charsetN, collationN, nullS, collumnDefaultN, commantS, columnLocation) + sqlS = fmt.Sprintf(" MODIFY COLUMN `%s` %s %s %s %s %s %s %s", curryColumn, columnDataType[0], charsetN, collationN, nullS, collumnDefaultN, commantS, columnLocation) case "drop": - sqlS = fmt.Sprintf(" drop column `%s` ", curryColumn) + sqlS = fmt.Sprintf(" DROP COLUMN `%s` ", curryColumn) } return sqlS } func (my *MysqlDataAbnormalFixStruct) FixAlterColumnSqlGenerate(modifyColumn []string, logThreadSeq int64) []string { var alterSql []string if len(modifyColumn) > 0 { - alterSql = append(alterSql, fmt.Sprintf("alter table `%s`.`%s` %s;", my.Schema, my.Table, strings.Join(modifyColumn, ","))) + alterSql = append(alterSql, fmt.Sprintf("ALTER TABLE `%s`.`%s` %s;", my.Schema, my.Table, strings.Join(modifyColumn, ","))) } return alterSql } diff --git a/MySQL/my_global_point.go b/MySQL/my_global_point.go index c5d00bc77ce9a6264f49cf774b872bd764a1fe57..2e9c4ab1088a89c1e888c47564c84575614973d3 100644 --- a/MySQL/my_global_point.go +++ b/MySQL/my_global_point.go @@ -81,7 +81,7 @@ func (my *GlobalCS) sessionRR(logThreadSeq int) ([]*sql.DB, error) { global.Wlog.Error(vlog) return nil, err } - strsql := "set session wait_timeout=86400;" + strsql := "SET SESSION wait_timeout=86400;" if _, err = tx.Exec(strsql); err != nil { vlog = fmt.Sprintf("(%d) MySQL DB exec sql %s fail. Error Info is {%s}.", logThreadSeq, strsql, err) global.Wlog.Error(vlog) @@ -93,7 +93,7 @@ func (my *GlobalCS) sessionRR(logThreadSeq int) ([]*sql.DB, error) { global.Wlog.Error(vlog) return nil, err } - strsql = "SET session sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));" + strsql = "SET SESSION sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));" if _, err = tx.Exec(strsql); err != nil { vlog = fmt.Sprintf("(%d) MySQL DB exec sql %s fail. Error Info is {%s}.", logThreadSeq, strsql, err) global.Wlog.Error(vlog) diff --git a/MySQL/my_query_table_date.go b/MySQL/my_query_table_date.go index 49b16e569ad243cbef1bf132b081a8a7323b5845..3f95505431d669db55514746050fd5f6d8317e26 100644 --- a/MySQL/my_query_table_date.go +++ b/MySQL/my_query_table_date.go @@ -19,7 +19,7 @@ func (my *QueryTable) QueryTableIndexColumnInfo(db *sql.DB, logThreadSeq int64) tableData []map[string]interface{} err error ) - strsql = fmt.Sprintf("select isc.COLUMN_NAME as columnName,isc.COLUMN_TYPE as columnType,isc.COLUMN_KEY as columnKey,isc.EXTRA as autoIncrement,iss.NON_UNIQUE as nonUnique,iss.INDEX_NAME as indexName,iss.SEQ_IN_INDEX as IndexSeq,isc.ORDINAL_POSITION as columnSeq from information_schema.columns isc inner join (select NON_UNIQUE,INDEX_NAME,SEQ_IN_INDEX,COLUMN_NAME from information_schema.STATISTICS where table_schema='%s' and table_name='%s') as iss on isc.column_name =iss.column_name where isc.table_schema='%s' and isc.table_name='%s';", my.Schema, my.Table, my.Schema, my.Table) + strsql = fmt.Sprintf("SELECT isc.COLUMN_NAME AS columnName, isc.COLUMN_TYPE AS columnType, isc.COLUMN_KEY AS columnKey,isc.EXTRA AS autoIncrement, iss.NON_UNIQUE AS nonUnique, iss.INDEX_NAME AS indexName, iss.SEQ_IN_INDEX AS IndexSeq, isc.ORDINAL_POSITION AS columnSeq FROM INFORMATION_SCHEMA.COLUMNS isc INNER JOIN (SELECT NON_UNIQUE, INDEX_NAME, SEQ_IN_INDEX, COLUMN_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s') AS iss ON isc.COLUMN_NAME=iss.COLUMN_NAME WHERE isc.TABLE_SCHEMA='%s' AND isc.TABLE_NAME='%s';", my.Schema, my.Table, my.Schema, my.Table) vlog = fmt.Sprintf("(%d) [%s] Generate a sql statement to query the index statistics of table %s.%s under the %s database.sql messige is {%s}", logThreadSeq, Event, my.Schema, my.Table, DBType, strsql) global.Wlog.Debug(vlog) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} @@ -56,7 +56,7 @@ func (my *QueryTable) IndexDisposF(queryData []map[string]interface{}, logThread for _, v := range queryData { currIndexName = fmt.Sprintf("%s", v["indexName"]) - if my.LowerCaseTableNames == "no" { + if my.CaseSensitiveObjectName == "no" { currIndexName = strings.ToUpper(fmt.Sprintf("%s", v["indexName"])) } @@ -137,17 +137,17 @@ func (my *QueryTable) TmpTableIndexColumnSelectDispos(logThreadSeq int64) map[st //根据索引列的多少，生成select 列条件，并生成列长度，为判断列是否为null或为空做判断 if len(columnName) == 1 { columnSelect["selectColumnName"] = strings.Join(columnName, "") - columnSelect["selectColumnLength"] = fmt.Sprintf("LENGTH(trim(%s)) as %s_length", strings.Join(columnName, ""), strings.Join(columnName, "")) + columnSelect["selectColumnLength"] = fmt.Sprintf("LENGTH(trim(%s)) AS %s_length", strings.Join(columnName, ""), strings.Join(columnName, "")) columnSelect["selectColumnLengthSlice"] = fmt.Sprintf("%s_length", strings.Join(columnName, "")) - columnSelect["selectColumnNull"] = fmt.Sprintf("%s is null ", strings.Join(columnName, "")) + columnSelect["selectColumnNull"] = fmt.Sprintf("%s IS NULL ", strings.Join(columnName, "")) columnSelect["selectColumnEmpty"] = fmt.Sprintf("%s = '' ", strings.Join(columnName, "")) } else if len(columnName) > 1 { columnSelect["selectColumnName"] = strings.Join(columnName, "/*column*/") var aa, bb, cc, dd, ee []string for i := range columnName { - aa = append(aa, fmt.Sprintf("LENGTH(trim(%s)) as %s_length", columnName[i], columnName[i])) + aa = append(aa, fmt.Sprintf("LENGTH(trim(%s)) AS %s_length", columnName[i], columnName[i])) bb = append(bb, fmt.Sprintf("%s_length", columnName[i])) - cc = append(cc, fmt.Sprintf("%s is null ", columnName[i])) + cc = append(cc, fmt.Sprintf("%s IS NULL ", columnName[i])) dd = append(dd, fmt.Sprintf("%s = '' ", columnName[i])) ee = append(ee, fmt.Sprintf("%s != '' ", columnName[i])) } @@ -172,7 +172,7 @@ func (my *QueryTable) TmpTableIndexColumnRowsCount(db *sql.DB, logThreadSeq int6 ) vlog = fmt.Sprintf("(%d) [%s] Start to query the total number of rows in the following table %s.%s of the %s database.", logThreadSeq, Event, my.Schema, my.Table, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select index_name AS INDEX_NAME,column_name AS columnName,cardinality as CARDINALITY from INFORMATION_SCHEMA.STATISTICS where TABLE_SCHEMA = '%s' and table_name = '%s' and SEQ_IN_INDEX=1", my.Schema, my.Table) + strsql = fmt.Sprintf("SELECT index_name AS INDEX_NAME, column_name AS columnName, cardinality as CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s' AND SEQ_IN_INDEX=1", my.Schema, my.Table) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return 0, err @@ -197,9 +197,9 @@ func (my *QueryTable) TmpTableIndexColumnRowsCount(db *sql.DB, logThreadSeq int6 } } if E != "" { - strsql = fmt.Sprintf("select sum(a.count) as sum from (select count(1) as count from `%s`.`%s` group by %s) a", my.Schema, my.Table, E) + strsql = fmt.Sprintf("SELECT SUM(a.count) AS sum FROM (SELECT COUNT(1) AS count FROM `%s`.`%s` GROUP BY %s) a", my.Schema, my.Table, E) } else { - strsql = fmt.Sprintf("select count(1) as sum from `%s`.`%s`", my.Schema, my.Table) + strsql = fmt.Sprintf("SELECT COUNT(1) AS sum FROM `%s`.`%s`", my.Schema, my.Table) } if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return 0, err @@ -262,7 +262,7 @@ func (my QueryTable) TmpTableColumnGroupDataDispos(db *sql.DB, where string, col } whereExist = where if where != "" { - whereExist = fmt.Sprintf("where %s ", where) + whereExist = fmt.Sprintf("WHERE %s ", where) if strings.Contains(version, "5.7") { whereExist, err = my.FloatTypeQueryDispos(db, where, logThreadSeq) if err != nil { @@ -270,7 +270,7 @@ func (my QueryTable) TmpTableColumnGroupDataDispos(db *sql.DB, where string, col } } } - strsql = fmt.Sprintf("select %s as columnName,count(1) as count from `%s`.`%s` %s group by %s", columnName, my.Schema, my.Table, whereExist, columnName) + strsql = fmt.Sprintf("SELECT %s AS columnName, COUNT(1) AS count FROM `%s`.`%s` %s GROUP BY %s", columnName, my.Schema, my.Table, whereExist, columnName) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err @@ -290,7 +290,7 @@ func (my *QueryTable) TableRows(db *sql.DB, logThreadSeq int64) (uint64, error) ) vlog = fmt.Sprintf("(%d) [%s] Start querying the statistical information of table %s.%s in the %s database and get the number of rows in the table", logThreadSeq, Event, my.Schema, my.Table, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select TABLE_ROWS as tableRows from information_schema.tables where table_schema='%s' and table_name ='%s'", my.Schema, my.Table) + strsql = fmt.Sprintf("SELECT TABLE_ROWS AS tableRows FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s'", my.Schema, my.Table) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return 0, err @@ -358,7 +358,7 @@ func (my *QueryTable) NoIndexGeneratingQueryCriteria(db *sql.DB, beginSeq uint64 } columnNameSeq = append(columnNameSeq, tmpcolumnName) } - strsql = fmt.Sprintf("select %s from `%s`.`%s` limit %d,%d", strings.Join(columnNameSeq, ","), my.Schema, my.Table, beginSeq, chanrowCount) + strsql = fmt.Sprintf("SELECT %s FROM `%s`.`%s` LIMIT %d,%d", strings.Join(columnNameSeq, ","), my.Schema, my.Table, beginSeq, chanrowCount) //if orderByColumn != "" { // strsql = fmt.Sprintf("select * from `%s`.`%s` order by %s limit %d,%d", my.Schema, my.Table, orderByColumn, beginSeq, chanrowCount) //} @@ -434,11 +434,11 @@ func (my *QueryTable) GeneratingQuerySql(db *sql.DB, logThreadSeq int64) (string return "", err } } else { - if !strings.HasPrefix(strings.TrimSpace(my.Sqlwhere), "where") { - my.Sqlwhere = fmt.Sprintf(" where %s ", my.Sqlwhere) + if !strings.HasPrefix(strings.TrimSpace(my.Sqlwhere), "WHERE") { + my.Sqlwhere = fmt.Sprintf(" WHERE %s ", my.Sqlwhere) } } - selectSql = fmt.Sprintf("select %s from `%s`.`%s` %s", queryColumn, my.Schema, my.Table, my.Sqlwhere) + selectSql = fmt.Sprintf("SELECT %s FROM `%s`.`%s` %s", queryColumn, my.Schema, my.Table, my.Sqlwhere) vlog = fmt.Sprintf("(%d) [%s] Complete the data query sql of table %s.%s in the %s database.", logThreadSeq, Event, my.Schema, my.Table, DBType) global.Wlog.Debug(vlog) return selectSql, nil diff --git a/MySQL/my_scheme_table_column.go b/MySQL/my_scheme_table_column.go index 9475e955c24fe39b139a260e2021d6d70187c248..ae8cc4d9ebcef062ee80d1eaab14c97264faf7b1 100644 --- a/MySQL/my_scheme_table_column.go +++ b/MySQL/my_scheme_table_column.go @@ -15,7 +15,7 @@ type QueryTable struct { IgnoreTable string Db *sql.DB Datafix string - LowerCaseTableNames string + CaseSensitiveObjectName string TmpTableFileName string ColumnName []string ChanrowCount int @@ -65,10 +65,10 @@ var ( user := strings.Split(fmt.Sprintf("%s", v["DEFINER"]), "@")[0] host := strings.Split(fmt.Sprintf("%s", v["DEFINER"]), "@")[1] if event == "Proc" { - tmpb[ROUTINE_NAME] = fmt.Sprintf("delimiter $\nCREATE DEFINER='%s'@'%s' PROCEDURE %s(%s) %s$ \ndelimiter ;", user, host, ROUTINE_NAME, tmpa[ROUTINE_NAME], strings.ReplaceAll(ROUTINE_DEFINITION, "\n", "")) + tmpb[ROUTINE_NAME] = fmt.Sprintf("DELIMITER $\nCREATE DEFINER='%s'@'%s' PROCEDURE %s(%s) %s$ \nDELIMITER ;", user, host, ROUTINE_NAME, tmpa[ROUTINE_NAME], strings.ReplaceAll(ROUTINE_DEFINITION, "\n", "")) } if event == "Func" { - tmpb[ROUTINE_NAME] = fmt.Sprintf("delimiter $\nCREATE DEFINER='%s'@'%s' FUNCTION %s(%s) %s$ \ndelimiter ;", user, host, ROUTINE_NAME, tmpa[ROUTINE_NAME], strings.ReplaceAll(ROUTINE_DEFINITION, "\n", "")) + tmpb[ROUTINE_NAME] = fmt.Sprintf("DELIMITER $\nCREATE DEFINER='%s'@'%s' FUNCTION %s(%s) %s$ \nDELIMITER ;", user, host, ROUTINE_NAME, tmpa[ROUTINE_NAME], strings.ReplaceAll(ROUTINE_DEFINITION, "\n", "")) } } return tmpb @@ -88,9 +88,9 @@ func (my *QueryTable) DatabaseNameList(db *sql.DB, logThreadSeq int64) (map[stri Event = "Q_Schema_Table_List" ) excludeSchema := fmt.Sprintf("'information_Schema','performance_Schema','sys','mysql'") - vlog = fmt.Sprintf("(%d) [%s] Start to query the metadata of the %s database and obtain library and table information.", logThreadSeq, Event, DBType) + strsql = fmt.Sprintf("SELECT TABLE_SCHEMA AS databaseName, TABLE_NAME AS tableName FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA NOT IN (%s);", excludeSchema) + vlog = fmt.Sprintf("(%d) [%s] Start to query the metadata of the %s database and obtain library and table information. SQL: {%s}", logThreadSeq, Event, DBType, strsql) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select TABLE_SCHEMA as databaseName,TABLE_NAME as tableName from information_Schema.TABLES where TABLE_SCHEMA not in (%s);", excludeSchema) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err @@ -102,7 +102,7 @@ func (my *QueryTable) DatabaseNameList(db *sql.DB, logThreadSeq int64) (map[stri for i := range tableData { var ga string gd, gt := fmt.Sprintf("%v", tableData[i]["databaseName"]), fmt.Sprintf("%v", tableData[i]["tableName"]) - if my.LowerCaseTableNames == "no" { + if my.CaseSensitiveObjectName == "no" { gd = strings.ToUpper(gd) gt = strings.ToUpper(gt) } @@ -124,8 +124,7 @@ func (my *QueryTable) TableColumnName(db *sql.DB, logThreadSeq int64) ([]map[str ) vlog = fmt.Sprintf("(%d) [%s] Start querying the metadata information of table %s.%s in the %s database and get all the column names", logThreadSeq, Event, my.Schema, my.Table, DBType) global.Wlog.Debug(vlog) - //strsql = fmt.Sprintf("select COLUMN_NAME as columnName from information_Schema.columns where TABLE_Schema='%s' and TABLE_NAME='%s' order by ORDINAL_POSITION;", my.Schema, my.Table) - strsql = fmt.Sprintf("select COLUMN_NAME as columnName,COLUMN_TYPE as columnType,IS_NULLABLE as isNull,CHARACTER_SET_NAME as charset,COLLATION_NAME as collationName,COLUMN_COMMENT as columnComment,COLUMN_DEFAULT as columnDefault from information_Schema.columns where TABLE_Schema='%s' and TABLE_NAME='%s' order by ORDINAL_POSITION", my.Schema, my.Table) + strsql = fmt.Sprintf("SELECT COLUMN_NAME AS columnName, COLUMN_TYPE AS columnType, IS_NULLABLE AS isNull, CHARACTER_SET_NAME AS charset, COLLATION_NAME AS collationName, COLUMN_COMMENT AS columnComment, COLUMN_DEFAULT AS columnDefault FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s' ORDER BY ORDINAL_POSITION", my.Schema, my.Table) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { if err != nil { @@ -152,7 +151,7 @@ func (my *QueryTable) DatabaseVersion(db *sql.DB, logThreadSeq int64) (string, e Event = "Q_M_Versions" ) vlog = fmt.Sprintf("(%d) [%s] Start querying the version information of the %s database", logThreadSeq, Event, DBType) - strsql = fmt.Sprintf("select version()") + strsql = fmt.Sprintf("SELECT VERSION() AS VERSION") dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if rows, err = dispos.DBSQLforExec(strsql); err != nil { if err != nil { @@ -168,7 +167,7 @@ func (my *QueryTable) DatabaseVersion(db *sql.DB, logThreadSeq int64) (string, e return "", nil } for _, i := range a { - if cc, ok := i["version()"]; ok { + if cc, ok := i["VERSION"]; ok { version = fmt.Sprintf("%v", cc) break } @@ -210,7 +209,7 @@ func (my *QueryTable) GlobalAccessPri(db *sql.DB, logThreadSeq int64) (bool, err globalPriS = append(globalPriS, k) } //获取当前匹配的用户 - strsql = fmt.Sprintf("select current_user() as user;") + strsql = fmt.Sprintf("SELECT CURRENT_USER() AS user;") dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if rows, err = dispos.DBSQLforExec(strsql); err != nil { return false, err @@ -228,7 +227,7 @@ func (my *QueryTable) GlobalAccessPri(db *sql.DB, logThreadSeq int64) (bool, err //查找全局权限 类似于grant all privileges on *.* 或 grant select on *.* vlog = fmt.Sprintf("(%d) [%s] Query the current %s DB global dynamic grants permission, to query it...", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select PRIVILEGE_TYPE as privileges from information_schema.USER_PRIVILEGES where PRIVILEGE_TYPE in('%s') and grantee = \"%s\";", strings.Join(globalPriS, "','"), currentUser) + strsql = fmt.Sprintf("SELECT PRIVILEGE_TYPE AS privileges FROM INFORMATION_SCHEMA.USER_PRIVILEGES WHERE PRIVILEGE_TYPE IN('%s') AND GRANTEE=\"%s\";", strings.Join(globalPriS, "','"), currentUser) if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return false, err } @@ -295,7 +294,7 @@ func (my *QueryTable) TableAccessPriCheck(db *sql.DB, checkTableList []string, d //校验库.表由切片改为map for _, AA := range checkTableList { newCheckTableList[AA]++ - if my.LowerCaseTableNames == "no" { + if my.CaseSensitiveObjectName == "no" { newCheckTableList[strings.ToUpper(AA)]++ } } @@ -303,13 +302,13 @@ func (my *QueryTable) TableAccessPriCheck(db *sql.DB, checkTableList []string, d for _, aa := range checkTableList { if strings.Contains(aa, ".") { A[strings.Split(aa, ".")[0]]++ - if my.LowerCaseTableNames == "no" { + if my.CaseSensitiveObjectName == "no" { A[strings.ToUpper(strings.Split(aa, ".")[0])]++ } } } //获取当前匹配的用户 - strsql = fmt.Sprintf("select current_user() as user;") + strsql = fmt.Sprintf("SELECT CURRENT_USER() AS user;") dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err @@ -322,7 +321,7 @@ func (my *QueryTable) TableAccessPriCheck(db *sql.DB, checkTableList []string, d //查找全局权限 类似于grant all privileges on *.* 或 grant select on *.* vlog = fmt.Sprintf("(%d) [%s] Query the current %s DB global dynamic grants permission, to query it...", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select PRIVILEGE_TYPE as privileges from information_schema.USER_PRIVILEGES where PRIVILEGE_TYPE in('%s') and grantee = \"%s\";", strings.Join(globalPriS, "','"), currentUser) + strsql = fmt.Sprintf("SELECT PRIVILEGE_TYPE AS privileges FROM INFORMATION_SCHEMA.USER_PRIVILEGES WHERE PRIVILEGE_TYPE IN('%s') AND GRANTEE=\"%s\";", strings.Join(globalPriS, "','"), currentUser) if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err } @@ -349,7 +348,7 @@ func (my *QueryTable) TableAccessPriCheck(db *sql.DB, checkTableList []string, d for AC, _ := range A { var cc []string var intseq int - strsql = fmt.Sprintf("select TABLE_SCHEMA as databaseName,PRIVILEGE_TYPE as privileges from information_schema.schema_PRIVILEGES where PRIVILEGE_TYPE in ('%s') and TABLE_SCHEMA = '%s' and grantee = \"%s\";", strings.Join(globalPriS, "','"), AC, currentUser) + strsql = fmt.Sprintf("SELECT TABLE_SCHEMA AS databaseName, PRIVILEGE_TYPE AS privileges FROM INFORMATION_SCHEMA.SCHEMA_PRIVILEGES WHERE PRIVILEGE_TYPE IN('%s') AND TABLE_SCHEMA='%s' AND GRANTEE=\"%s\";", strings.Join(globalPriS, "','"), AC, currentUser) if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err } @@ -387,13 +386,13 @@ func (my *QueryTable) TableAccessPriCheck(db *sql.DB, checkTableList []string, d var DM = make(map[string]int) for _, D := range checkTableList { DM[D]++ - if my.LowerCaseTableNames == "no" { + if my.CaseSensitiveObjectName == "no" { DM[strings.ToUpper(D)]++ } } for B, _ := range A { //按照每个库，查询table pri权限 - strsql = fmt.Sprintf("select table_name as tableName,PRIVILEGE_TYPE as privileges from information_schema.table_PRIVILEGES where PRIVILEGE_TYPE in('%s') and TABLE_SCHEMA = '%s' and grantee = \"%s\";", strings.Join(globalPriS, "','"), B, currentUser) + strsql = fmt.Sprintf("SELECT TABLE_NAME AS tableName, PRIVILEGE_TYPE AS privileges FROM INFORMATION_SCHEMA.TABLE_PRIVILEGES WHERE PRIVILEGE_TYPE IN('%s') AND TABLE_SCHEMA='%s' AND GRANTEE=\"%s\";", strings.Join(globalPriS, "','"), B, currentUser) if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err } @@ -411,7 +410,7 @@ func (my *QueryTable) TableAccessPriCheck(db *sql.DB, checkTableList []string, d for _, C := range tablePri { var E string E = fmt.Sprintf("%s.%s", B, C["tableName"]) - if my.LowerCaseTableNames == "no" { + if my.CaseSensitiveObjectName == "no" { E = strings.ToUpper(fmt.Sprintf("%s.%s", B, C["tableName"])) } if E != N { @@ -453,7 +452,7 @@ func (my *QueryTable) TableAllColumn(db *sql.DB, logThreadSeq int64) ([]map[stri ) vlog = fmt.Sprintf("(%d) [%s] Start to query the metadata of all the columns of table %s.%s in the %s database", logThreadSeq, Event, my.Schema, my.Table, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select COLUMN_NAME as columnName ,COLUMN_TYPE as dataType,ORDINAL_POSITION as columnSeq,IS_NULLABLE as isNull from information_Schema.columns where table_Schema= '%s' and table_name='%s' order by ORDINAL_POSITION;", my.Schema, my.Table) + strsql = fmt.Sprintf("SELECT COLUMN_NAME AS columnName, COLUMN_TYPE AS dataType, ORDINAL_POSITION AS columnSeq, IS_NULLABLE AS isNull FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s' ORDER BY ORDINAL_POSITION;", my.Schema, my.Table) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err @@ -637,7 +636,7 @@ func (my *QueryTable) TableIndexChoice(queryData []map[string]interface{}, logTh indexChoice[k] = v } } - f := my.keyChoiceDispos(multiseriateIndexColumnMap, "mui") + f := my.keyChoiceDispos(multiseriateIndexColumnMap, "mul") for k, v := range f { if len(v) > 0 { indexChoice[k] = v @@ -658,7 +657,7 @@ func (my *QueryTable) Trigger(db *sql.DB, logThreadSeq int64) (map[string]string ) vlog = fmt.Sprintf("(%d) [%s] Start to query the trigger information under the %s database.", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select TRIGGER_NAME as triggerName,EVENT_OBJECT_TABLE as tableName from INFORMATION_SCHEMA.TRIGGERS where TRIGGER_SCHEMA in ('%s');", my.Schema) + strsql = fmt.Sprintf("SELECT TRIGGER_NAME AS triggerName, EVENT_OBJECT_TABLE AS tableName FROM INFORMATION_SCHEMA.TRIGGERS WHERE TRIGGER_SCHEMA IN('%s');", my.Schema) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err @@ -668,7 +667,7 @@ func (my *QueryTable) Trigger(db *sql.DB, logThreadSeq int64) (map[string]string return nil, err } for _, v := range triggerName { - strsql = fmt.Sprintf("show create trigger %s.%s", my.Schema, v["triggerName"]) + strsql = fmt.Sprintf("SHOW CREATE TRIGGER %s.%s", my.Schema, v["triggerName"]) if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err } @@ -714,7 +713,7 @@ func (my *QueryTable) Proc(db *sql.DB, logThreadSeq int64) (map[string]string, e ) vlog = fmt.Sprintf("(%d) [%s] Start to query the stored procedure information under the %s database.", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select SPECIFIC_SCHEMA,SPECIFIC_NAME,ORDINAL_POSITION,PARAMETER_MODE,PARAMETER_NAME,DTD_IDENTIFIER from information_schema.PARAMETERS where SPECIFIC_SCHEMA in ('%s') and ROUTINE_TYPE='PROCEDURE' order by ORDINAL_POSITION;", my.Schema) + strsql = fmt.Sprintf("SELECT SPECIFIC_SCHEMA, SPECIFIC_NAME, ORDINAL_POSITION, PARAMETER_MODE, PARAMETER_NAME, DTD_IDENTIFIER FROM INFORMATION_SCHEMA.PARAMETERS WHERE SPECIFIC_SCHEMA IN('%s') AND ROUTINE_TYPE='PROCEDURE' ORDER BY ORDINAL_POSITION;", my.Schema) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err @@ -723,7 +722,7 @@ func (my *QueryTable) Proc(db *sql.DB, logThreadSeq int64) (map[string]string, e if err != nil { return nil, err } - strsql = fmt.Sprintf("select ROUTINE_SCHEMA,ROUTINE_NAME,ROUTINE_DEFINITION,DEFINER from information_schema.ROUTINES where routine_schema in ('%s') and ROUTINE_TYPE='PROCEDURE';", my.Schema) + strsql = fmt.Sprintf("SELECT ROUTINE_SCHEMA, ROUTINE_NAME, ROUTINE_DEFINITION, DEFINER FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA IN('%s') AND ROUTINE_TYPE='PROCEDURE';", my.Schema) if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err } @@ -747,7 +746,7 @@ func (my *QueryTable) Func(db *sql.DB, logThreadSeq int64) (map[string]string, e ) vlog = fmt.Sprintf("(%d) [%s] Start to query the stored Func information under the %s database.", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select DEFINER,ROUTINE_NAME from information_schema.ROUTINES where routine_schema in ('%s') and ROUTINE_TYPE='FUNCTION';", my.Schema) + strsql = fmt.Sprintf("SELECT DEFINER, ROUTINE_NAME FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA IN('%s') AND ROUTINE_TYPE='FUNCTION';", my.Schema) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err @@ -793,7 +792,7 @@ func (my *QueryTable) Foreign(db *sql.DB, logThreadSeq int64) (map[string]string ) vlog = fmt.Sprintf("(%d) [%s] Start to query the Foreign information under the %s database.", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select CONSTRAINT_SCHEMA,TABLE_NAME from information_schema.referential_constraints where CONSTRAINT_SCHEMA in ('%s') and TABLE_NAME in ('%s');", my.Schema, my.Table) + strsql = fmt.Sprintf("SELECT CONSTRAINT_SCHEMA, TABLE_NAME FROM INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA IN('%s') AND TABLE_NAME IN('%s');", my.Schema, my.Table) //vlog = fmt.Sprintf("(%d) MySQL DB query table query Foreign info exec sql is {%s}", logThreadSeq, sqlStr) //global.Wlog.Debug(vlog) @@ -893,7 +892,7 @@ func (my *QueryTable) Partitions(db *sql.DB, logThreadSeq int64) (map[string]str ) vlog = fmt.Sprintf("(%d) [%s] Start to query the Partitions information under the %s database.", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select TABLE_SCHEMA,TABLE_NAME from information_schema.partitions where table_schema in ('%s') and TABLE_NAME in ('%s') and PARTITION_NAME <> '';", my.Schema, my.Table) + strsql = fmt.Sprintf("SELECT TABLE_SCHEMA, TABLE_NAME FROM INFORMATION_SCHEMA.PARTITIONS WHERE TABLE_SCHEMA IN('%s') AND TABLE_NAME IN('%s') AND PARTITION_NAME<>'';", my.Schema, my.Table) //vlog = fmt.Sprintf("(%d) MySQL DB query table query partitions info exec sql is {%s}", logThreadSeq, sqlStr) //global.Wlog.Debug(vlog) //sqlRows, err := db.Query(sqlStr) diff --git a/Oracle/or_data_fix_sql.go b/Oracle/or_data_fix_sql.go index 49bc13a35c7869be8067b5d969d658a2647a141c..2b807a6e43f8a59ae4a51ddc0653a644e4ca80c8 100644 --- a/Oracle/or_data_fix_sql.go +++ b/Oracle/or_data_fix_sql.go @@ -78,10 +78,10 @@ func (or *OracleDataAbnormalFixStruct) FixInsertSqlExec(db *sql.DB, sourceDrive if len(valuesNameSeq) > 0 { queryColumn := strings.Join(valuesNameSeq, ",") if or.DatafixType == "file" { - insertSql = fmt.Sprintf("insert into \"%s\".\"%s\" values(%s);", or.Schema, or.Table, queryColumn) + insertSql = fmt.Sprintf("INSERT INTO \"%s\".\"%s\" VALUES(%s);", or.Schema, or.Table, queryColumn) } if or.DatafixType == "table" { - insertSql = fmt.Sprintf("insert into \"%s\".\"%s\" values(%s)", or.Schema, or.Table, queryColumn) + insertSql = fmt.Sprintf("INSERT INTO \"%s\".\"%s\" VALUES(%s)", or.Schema, or.Table, queryColumn) } } return insertSql, nil @@ -105,7 +105,7 @@ func (or *OracleDataAbnormalFixStruct) FixDeleteSqlExec(db *sql.DB, sourceDrive global.Wlog.Debug(vlog) vlog = fmt.Sprintf("(%d) MySQL DB check table %s.%s Generate delete repair statement based on unique index.", logThreadSeq, or.Schema, or.Table) global.Wlog.Debug(vlog) - if or.IndexType == "mui" { + if or.IndexType == "mul" { var FB, AS []string for _, i := range colData { FB = append(FB, i["columnName"]) @@ -113,16 +113,16 @@ func (or *OracleDataAbnormalFixStruct) FixDeleteSqlExec(db *sql.DB, sourceDrive rowData := strings.ReplaceAll(or.RowData, "/*go actions columnData*//*go actions columnData*/", "/*go actions columnData*/greatdbNull/*go actions columnData*/") for k, v := range strings.Split(rowData, "/*go actions columnData*/") { if v == "" { - AS = append(AS, fmt.Sprintf(" %s is null ", FB[k])) + AS = append(AS, fmt.Sprintf(" %s IS NULL ", FB[k])) } else if v == "" { AS = append(AS, fmt.Sprintf(" %s = ''", FB[k])) } else if v == acc["double"] { - AS = append(AS, fmt.Sprintf(" concat(%s,'') = '%s'", FB[k], v)) + AS = append(AS, fmt.Sprintf(" CONCAT(%s,'') = '%s'", FB[k], v)) } else { AS = append(AS, fmt.Sprintf(" %s = '%s' ", FB[k], v)) } } - deleteSqlWhere = strings.Join(AS, " and ") + deleteSqlWhere = strings.Join(AS, " AND ") } if or.IndexType == "pri" || or.IndexType == "uni" { var FB []string @@ -138,25 +138,25 @@ func (or *OracleDataAbnormalFixStruct) FixDeleteSqlExec(db *sql.DB, sourceDrive for l, I := range FB { if I == strconv.Itoa(k+1) { if v == "" { - AS = append(AS, fmt.Sprintf(" %s is null ", or.IndexColumn[l])) + AS = append(AS, fmt.Sprintf(" %s IS NULL ", or.IndexColumn[l])) } else if v == "" { AS = append(AS, fmt.Sprintf(" %s = ''", FB[k])) } else if v == acc["double"] { - AS = append(AS, fmt.Sprintf(" concat(%s,'') = '%s'", or.IndexColumn[l], v)) + AS = append(AS, fmt.Sprintf(" CONCAT(%s,'') = '%s'", or.IndexColumn[l], v)) } else { AS = append(AS, fmt.Sprintf(" %s = '%s' ", or.IndexColumn[l], v)) } } - deleteSqlWhere = strings.Join(AS, " and ") + deleteSqlWhere = strings.Join(AS, " AND ") } } } if len(deleteSqlWhere) > 0 { if or.DatafixType == "file" { - deleteSql = fmt.Sprintf("delete from \"%s\".\"%s\" where %s;", or.Schema, or.Table, deleteSqlWhere) + deleteSql = fmt.Sprintf("DELETE FROM \"%s\".\"%s\" WHERE %s;", or.Schema, or.Table, deleteSqlWhere) } if or.DatafixType == "table" { - deleteSql = fmt.Sprintf("delete from \"%s\".\"%s\" where %s", or.Schema, or.Table, deleteSqlWhere) + deleteSql = fmt.Sprintf("DELETE FROM \"%s\".\"%s\" WHERE %s", or.Schema, or.Table, deleteSqlWhere) } } return deleteSql, nil @@ -173,22 +173,22 @@ func (or *OracleDataAbnormalFixStruct) FixAlterIndexSqlExec(e, f []string, si ma } switch or.IndexType { case "pri": - strsql = fmt.Sprintf("alter table %s.%s add primary key(`%s`);", or.Schema, or.Table, strings.Join(c, "`,`")) + strsql = fmt.Sprintf("ALTER TABLE %s.%s ADD PRIMARY KEY(`%s`);", or.Schema, or.Table, strings.Join(c, "`,`")) case "uni": - strsql = fmt.Sprintf("alter table %s.%s add unique index %s(`%s`);", or.Schema, or.Table, v, strings.Join(c, "`,`")) + strsql = fmt.Sprintf("ALTER TABLE %s.%s ADD UNIQUE INDEX %s(`%s`);", or.Schema, or.Table, v, strings.Join(c, "`,`")) case "mul": - strsql = fmt.Sprintf("alter table %s.%s add index %s(`%s`);", or.Schema, or.Table, v, strings.Join(c, "`,`")) + strsql = fmt.Sprintf("ALTER TABLE %s.%s ADD INDEX %s(`%s`);", or.Schema, or.Table, v, strings.Join(c, "`,`")) } sqlS = append(sqlS, strsql) } for _, v := range f { switch or.IndexType { case "pri": - strsql = fmt.Sprintf("alter table %s.%s drop primary key;", or.Schema, or.Table) + strsql = fmt.Sprintf("ALTER TABLE %s.%s DROP PRIMARY KEY;", or.Schema, or.Table) case "uni": - strsql = fmt.Sprintf("alter table %s.%s drop index %s;", or.Schema, or.Table, v) + strsql = fmt.Sprintf("ALTER TABLE %s.%s DROP INDEX %s;", or.Schema, or.Table, v) case "mul": - strsql = fmt.Sprintf("alter table %s.%s drop index %s;", or.Schema, or.Table, v) + strsql = fmt.Sprintf("ALTER TABLE %s.%s DROP INDEX %s;", or.Schema, or.Table, v) } sqlS = append(sqlS, strsql) } @@ -227,7 +227,7 @@ func (or *OracleDataAbnormalFixStruct) FixAlterColumnSqlDispos(alterType string, func (or *OracleDataAbnormalFixStruct) FixAlterColumnSqlGenerate(modifyColumn []string, logThreadSeq int64) []string { var alterSql []string if len(modifyColumn) > 0 { - alterSql = append(alterSql, fmt.Sprintf("alter table `%s`.`%s` %s", or.Schema, or.Table, strings.Join(modifyColumn, ","))) + alterSql = append(alterSql, fmt.Sprintf("ALTER TABLE `%s`.`%s` %s", or.Schema, or.Table, strings.Join(modifyColumn, ","))) } return alterSql } diff --git a/Oracle/or_global_point.go b/Oracle/or_global_point.go index 9890fcdd4497d23d8d3fbfd895b81de87be94a71..53d50ea0f990b991bd60857a578475bfddb0bc32 100644 --- a/Oracle/or_global_point.go +++ b/Oracle/or_global_point.go @@ -16,7 +16,7 @@ type GlobalCS struct { } func (or *GlobalCS) flushTable(db *sql.DB, logThreadSeq int) error { - sqlstr := fmt.Sprintf("alter system checkpoint") + sqlstr := fmt.Sprintf("ALTER SYSTEM CHECKPOINT") alog := fmt.Sprintf("(%d) Oracle DB alter system checkpoint...", logThreadSeq) global.Wlog.Info(alog) if _, err := db.Exec(sqlstr); err != nil { @@ -113,7 +113,7 @@ func (or *GlobalCS) globalConsistencyPoint(db *sql.DB, logThreadSeq int) (map[st var position string var rows *sql.Rows var globalPoint = make(map[string]string) - sqlstr := fmt.Sprintf("select current_scn as \"globalScn\" from v$database") + sqlstr := fmt.Sprintf("SELECT CURRENT_SCN AS \"globalScn\" FROM v$database") alog := fmt.Sprintf("(%d) Oracle DB start select current_scn from v$database...", logThreadSeq) global.Wlog.Info(alog) rows, err := db.Query(sqlstr) diff --git a/Oracle/or_query_table_date.go b/Oracle/or_query_table_date.go index 4388db1f98db4f081c72cf071dccc69d9fa2fda5..8c4f8cfa7c3d20416f3bc150439fd84b719c5d72 100644 --- a/Oracle/or_query_table_date.go +++ b/Oracle/or_query_table_date.go @@ -20,7 +20,7 @@ func (or *QueryTable) QueryTableIndexColumnInfo(db *sql.DB, logThreadSeq int64) tableData []map[string]interface{} err error ) - strsql = fmt.Sprintf("select c.COLUMN_NAME as \"columnName\",decode(c.DATA_TYPE,'DATE',c.data_type,c.DATA_TYPE || '(' || c.data_LENGTH || ')') as \"columnType\", decode(co.constraint_type, 'P','1','0') as \"columnKey\",i.UNIQUENESS as \"nonUnique\", ic.INDEX_NAME as \"indexName\", ic.COLUMN_POSITION as \"IndexSeq\", c.COLUMN_ID as \"columnSeq\" from all_tab_cols c inner join all_ind_columns ic on c.TABLE_NAME = ic.TABLE_NAME and c.OWNER = ic.INDEX_OWNER and c.COLUMN_NAME = ic.COLUMN_NAME inner join all_indexes i on ic.INDEX_OWNER = i.OWNER and ic.INDEX_NAME = i.INDEX_NAME and ic.TABLE_NAME = i.TABLE_NAME left join all_constraints co on co.owner = c.owner and co.table_name = c.table_name and co.index_name = i.index_name where c.OWNER = '%s' and c.TABLE_NAME = '%s' ORDER BY I.INDEX_NAME, ic.COLUMN_POSITION", strings.ToUpper(or.Schema), or.Table) + strsql = fmt.Sprintf("SELECT c.COLUMN_NAME AS \"columnName\", DECODE(c.DATA_TYPE, 'DATE', c.data_type, c.DATA_TYPE || '(' || c.data_LENGTH || ')') AS \"columnType\", DECODE(co.constraint_type, 'P', '1', '0') AS \"columnKey\", i.UNIQUENESS AS \"nonUnique\", ic.INDEX_NAME AS \"indexName\", ic.COLUMN_POSITION AS \"IndexSeq\", c.COLUMN_ID AS \"columnSeq\" FROM all_tab_cols c INNER JOIN all_ind_columns ic ON c.TABLE_NAME=ic.TABLE_NAME AND c.OWNER=ic.INDEX_OWNER AND c.COLUMN_NAME=ic.COLUMN_NAME INNER JOIN all_indexes i ON ic.INDEX_OWNER=i.OWNER AND ic.INDEX_NAME=i.INDEX_NAME AND ic.TABLE_NAME=i.TABLE_NAME LEFT JOIN all_constraints co ON co.owner=c.owner AND co.table_name=c.table_name AND co.index_name=i.index_name WHERE c.OWNER='%s' AND c.TABLE_NAME='%s' ORDER BY I.INDEX_NAME, ic.COLUMN_POSITION", strings.ToUpper(or.Schema), or.Table) vlog = fmt.Sprintf("(%d) [%s] Generate a sql statement to query the index statistics of table %s.%s under the %s database.sql messige is {%s}", logThreadSeq, Event, or.Schema, or.Table, DBType, strsql) global.Wlog.Debug(vlog) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} @@ -57,7 +57,7 @@ func (or *QueryTable) IndexDisposF(queryData []map[string]interface{}, logThread for _, v := range queryData { currIndexName = fmt.Sprintf("%s", v["indexName"]) - if or.LowerCaseTableNames == "no" { + if or.CaseSensitiveObjectName == "no" { currIndexName = strings.ToUpper(fmt.Sprintf("%s", v["indexName"])) } @@ -138,7 +138,7 @@ func (or *QueryTable) TmpTableIndexColumnSelectDispos(logThreadSeq int64) map[st //根据索引列的多少，生成select 列条件，并生成列长度，为判断列是否为null或为空做判断 if len(columnName) == 1 { columnSelect["selectColumnName"] = strings.Join(columnName, "") - columnSelect["selectColumnLength"] = fmt.Sprintf("LENGTH(trim(%s)) as %s_length", strings.Join(columnName, ""), strings.Join(columnName, "")) + columnSelect["selectColumnLength"] = fmt.Sprintf("LENGTH(trim(%s)) AS %s_length", strings.Join(columnName, ""), strings.Join(columnName, "")) columnSelect["selectColumnLengthSlice"] = fmt.Sprintf("%s_length", strings.Join(columnName, "")) columnSelect["selectColumnNull"] = fmt.Sprintf("%s is null ", strings.Join(columnName, "")) columnSelect["selectColumnEmpty"] = fmt.Sprintf("%s = '' ", strings.Join(columnName, "")) @@ -147,7 +147,7 @@ func (or *QueryTable) TmpTableIndexColumnSelectDispos(logThreadSeq int64) map[st columnSelect["selectColumnName"] = strings.Join(columnName, "/*column*/") var aa, bb, cc, dd []string for i := range columnName { - aa = append(aa, fmt.Sprintf("LENGTH(trim(%s)) as %s_length", columnName[i], columnName[i])) + aa = append(aa, fmt.Sprintf("LENGTH(trim(%s)) AS %s_length", columnName[i], columnName[i])) bb = append(bb, fmt.Sprintf("%s_length", columnName[i])) cc = append(cc, fmt.Sprintf("%s is null ", columnName[i])) dd = append(dd, fmt.Sprintf("%s = '' ", columnName[i])) @@ -172,7 +172,7 @@ func (or *QueryTable) TmpTableIndexColumnRowsCount(db *sql.DB, logThreadSeq int6 ) vlog = fmt.Sprintf("(%d) [%s] Start to query the total number of rows in the following table %s.%s of the %s database.", logThreadSeq, Event, or.Schema, or.Table, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select count(1) as \"sum\" from \"%s\".\"%s\"", or.Schema, or.Table) + strsql = fmt.Sprintf("SELECT COUNT(1) AS \"sum\" FROM \"%s\".\"%s\"", or.Schema, or.Table) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return 0, err @@ -205,7 +205,7 @@ func (or *QueryTable) TmpTableColumnGroupDataDispos(db *sql.DB, where string, co if where != "" { whereExist = fmt.Sprintf("where %s ", where) } - strsql = fmt.Sprintf("select %s as \"columnName\",count(1) as \"count\" from \"%s\".\"%s\" %s group by %s order by %s", columnName, or.Schema, or.Table, whereExist, columnName, columnName) + strsql = fmt.Sprintf("SELECT %s AS \"columnName\", COUNT(1) AS \"count\" FROM \"%s\".\"%s\" %s GROUP BY %s ORDER BY %s", columnName, or.Schema, or.Table, whereExist, columnName, columnName) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err @@ -226,7 +226,7 @@ func (or *QueryTable) TableRows(db *sql.DB, logThreadSeq int64) (uint64, error) ) vlog = fmt.Sprintf("(%d) [%s] Start to query the total number of rows in the following table %s.%s of the %s database.", logThreadSeq, Event, or.Schema, or.Table, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select count(1) as \"sum\" from \"%s\".\"%s\"", or.Schema, or.Table) + strsql = fmt.Sprintf("SELECT COUNT(1) AS \"sum\" FROM \"%s\".\"%s\"", or.Schema, or.Table) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return 0, err @@ -356,10 +356,10 @@ func (or *QueryTable) GeneratingQuerySql(db *sql.DB, logThreadSeq int64) (string nu := "0" tmpcolumnName := fmt.Sprintf("\"%s\"", i["columnName"]) if strings.ToUpper(i["dataType"]) == "DATE" { - tmpcolumnName = fmt.Sprintf("to_char(%s,'YYYY-MM-DD HH24:MI:SS')", tmpcolumnName) + tmpcolumnName = fmt.Sprintf("TO_CHAR(%s,'YYYY-MM-DD HH24:MI:SS')", tmpcolumnName) } if strings.Contains(strings.ToUpper(i["dataType"]), "TIMESTAMP") { - tmpcolumnName = fmt.Sprintf("to_char(%s,'YYYY-MM-DD HH24:MI:SS')", tmpcolumnName) + tmpcolumnName = fmt.Sprintf("TO_CHAR(%s,'YYYY-MM-DD HH24:MI:SS')", tmpcolumnName) } if strings.HasPrefix(strings.ToUpper(i["dataType"]), "NUMBER(") { dianAfter := strings.ReplaceAll(strings.Split(i["dataType"], ",")[1], ")", "") @@ -374,16 +374,16 @@ func (or *QueryTable) GeneratingQuerySql(db *sql.DB, logThreadSeq int64) (string tmpb = append(tmpb, mu) } if bb == 0 { - tmpcolumnName = fmt.Sprintf("to_char(%s,'FM%s0')", tmpcolumnName, strings.Join(tmpb, "")) + tmpcolumnName = fmt.Sprintf("TO_CHAR(%s,'FM%s0')", tmpcolumnName, strings.Join(tmpb, "")) } else { - tmpcolumnName = fmt.Sprintf("to_char(%s,'FM%s0.%s')", tmpcolumnName, strings.Join(tmpb, ""), strings.Join(tmpa, "")) + tmpcolumnName = fmt.Sprintf("TO_CHAR(%s,'FM%s0.%s')", tmpcolumnName, strings.Join(tmpb, ""), strings.Join(tmpa, "")) } } columnNameSeq = append(columnNameSeq, tmpcolumnName) } queryColumn := strings.Join(columnNameSeq, ",") //sqlstr := fmt.Sprintf("select %s from \"%s\".\"%s\" as of scn %s where %s", queryColumn, schema, table, oracleScn, sqlWhere) - selectSql = fmt.Sprintf("select %s from \"%s\".\"%s\" where %s", queryColumn, strings.ToUpper(or.Schema), or.Table, or.Sqlwhere) + selectSql = fmt.Sprintf("SELECT %s FROM \"%s\".\"%s\" WHERE %s", queryColumn, strings.ToUpper(or.Schema), or.Table, or.Sqlwhere) vlog = fmt.Sprintf("(%d) [%s] Complete the data query sql of table %s.%s in the %s database.", logThreadSeq, Event, or.Schema, or.Table, DBType) global.Wlog.Debug(vlog) return selectSql, nil diff --git a/Oracle/or_scheme_table_column.go b/Oracle/or_scheme_table_column.go index 9a07fb185383f7265e2dd0e885e83381a4a5b8eb..7af6827ab49b9de033a4e78b582013e64ca0ad46 100644 --- a/Oracle/or_scheme_table_column.go +++ b/Oracle/or_scheme_table_column.go @@ -13,7 +13,7 @@ type QueryTable struct { Table string Db *sql.DB Datafix string - LowerCaseTableNames string + CaseSensitiveObjectName string TmpTableFileName string ColumnName []string ChanrowCount int @@ -44,7 +44,7 @@ func (or *QueryTable) DatabaseNameList(db *sql.DB, logThreadSeq int64) (map[stri excludeSchema = fmt.Sprintf("'SYS','OUTLN','SYSTEM','DBSNMP','APPQOSSYS','WMSYS','EXFSYS','CTXSYS','XDB','ORDDATA','ORDSYS','MDSYS','OLAPSYS','SYSMAN','FLOWS_FILES','APEX_030200','OWBSYS','HR','OE','SH','IX','PM'") vlog = fmt.Sprintf("(%d) [%s] Start to query the metadata of the %s database and obtain library and table information.", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("SELECT owner as \"databaseName\",table_name as \"tableName\" FROM DBA_TABLES WHERE OWNER not in (%s)", excludeSchema) + strsql = fmt.Sprintf("SELECT owner AS \"databaseName\", table_name AS \"tableName\" FROM DBA_TABLES WHERE OWNER NOT IN(%s)", excludeSchema) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} rows, err := dispos.DBSQLforExec(strsql) if err != nil { @@ -58,7 +58,7 @@ func (or *QueryTable) DatabaseNameList(db *sql.DB, logThreadSeq int64) (map[stri for i := range tableData { var ga string gd, gt := fmt.Sprintf("%v", tableData[i]["databaseName"]), fmt.Sprintf("%v", tableData[i]["tableName"]) - if or.LowerCaseTableNames == "no" { + if or.CaseSensitiveObjectName == "no" { gd = strings.ToUpper(gd) gt = strings.ToUpper(gt) } @@ -80,7 +80,7 @@ func (or *QueryTable) TableColumnName(db *sql.DB, logThreadSeq int64) ([]map[str ) vlog = fmt.Sprintf("(%d) [%s] Start querying the metadata information of table %s.%s in the %s database and get all the column names", logThreadSeq, Event, or.Schema, or.Table, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select tc.column_name as \"columnName\",decode(tc.data_type,'NUMBER',NVL2(DATA_PRECISION,'NUMBER(' || tc.DATA_PRECISION || ',' || tc.DATA_SCALE || ')','NUMBER'),'VARCHAR2','VARCHAR2(' || tc.DATA_LENGTH || ')','CHAR','CHAR(' || tc.DATA_LENGTH || ')','RAW','RAW(' || tc.DATA_LENGTH || ')',tc.DATA_TYPE) as \"columnType\",NULLABLE as \"isNull\",'','',to_nchar(cc.comments) as \"columnComment\",DATA_DEFAULT as \"columnDefault\" from dba_tab_columns tc join dba_col_comments cc on tc.OWNER = cc.owner and tc.TABLE_NAME = cc.table_name and tc.COLUMN_NAME = cc.column_name WHERE tc.owner = '%s' and tc.table_name = '%s' order by tc.COLUMN_ID", or.Schema, or.Table) + strsql = fmt.Sprintf("SELECT tc.column_name AS \"columnName\", DECODE(tc.data_type, 'NUMBER', NVL2(DATA_PRECISION, 'NUMBER(' || tc.DATA_PRECISION || ',' || tc.DATA_SCALE || ')', 'NUMBER'), 'VARCHAR2', 'VARCHAR2(' || tc.DATA_LENGTH || ')', 'CHAR', 'CHAR(' || tc.DATA_LENGTH || ')', 'RAW', 'RAW(' || tc.DATA_LENGTH || ')',tc.DATA_TYPE) AS \"columnType\", NULLABLE AS \"isNull\", '', '', TO_NCHAR(cc.comments) AS \"columnComment\", DATA_DEFAULT AS \"columnDefault\" FROM dba_tab_columns tc JOIN dba_col_comments cc ON tc.OWNER=cc.owner AND tc.TABLE_NAME=cc.table_name AND tc.COLUMN_NAME=cc.column_name WHERE tc.owner='%s' AND tc.table_name = '%s' ORDER BY tc.COLUMN_ID", or.Schema, or.Table) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { if err != nil { @@ -119,7 +119,7 @@ func (or *QueryTable) GlobalAccessPri(db *sql.DB, logThreadSeq int64) (bool, err //查找全局权限 类似于grant all privileges on *.* 或 grant select on *.* vlog = fmt.Sprintf("(%d) [%s] Query the current %s DB global dynamic grants permission, to query it...", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select PRIVILEGE as \"privileges\" from user_sys_privs where PRIVILEGE IN ('%s')", strings.Join(globalPriS, "','")) + strsql = fmt.Sprintf("SELECT PRIVILEGE AS \"privileges\" FROM user_sys_privs WHERE PRIVILEGE IN('%s')", strings.Join(globalPriS, "','")) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return false, err @@ -129,7 +129,7 @@ func (or *QueryTable) GlobalAccessPri(db *sql.DB, logThreadSeq int64) (bool, err } //权限缺失列表 if len(globalDynamic) == 0 { - strsql = fmt.Sprintf("SELECT PRIVILEGE as \"privileges\" FROM ROLE_SYS_PRIVS WHERE PRIVILEGE IN ('%s') group by PRIVILEGE", strings.Join(globalPriS, "','")) + strsql = fmt.Sprintf("SELECT PRIVILEGE AS \"privileges\" FROM ROLE_SYS_PRIVS WHERE PRIVILEGE IN('%s') GROUP BY PRIVILEGE", strings.Join(globalPriS, "','")) if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return false, err } @@ -219,7 +219,7 @@ func (or *QueryTable) TableAccessPriCheck(db *sql.DB, checkTableList []string, d //校验库.表由切片改为map for _, AA := range checkTableList { newCheckTableList[AA]++ - if or.LowerCaseTableNames == "no" { + if or.CaseSensitiveObjectName == "no" { newCheckTableList[strings.ToUpper(AA)]++ } } @@ -229,7 +229,7 @@ func (or *QueryTable) TableAccessPriCheck(db *sql.DB, checkTableList []string, d for _, aa := range checkTableList { if strings.Contains(aa, ".") { A[strings.Split(aa, ".")[0]]++ - if or.LowerCaseTableNames == "no" { + if or.CaseSensitiveObjectName == "no" { A[strings.ToUpper(strings.Split(aa, ".")[0])]++ } } @@ -238,7 +238,7 @@ func (or *QueryTable) TableAccessPriCheck(db *sql.DB, checkTableList []string, d //查找全局权限 类似于grant all privileges on *.* 或 grant select on *.* vlog = fmt.Sprintf("(%d) [%s] Query the current %s DB global dynamic grants permission, to query it...", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("SELECT PRIVILEGE as \"privileges\" FROM ROLE_SYS_PRIVS WHERE PRIVILEGE IN ('%s') group by PRIVILEGE", strings.Join(priAllTableS, "','")) + strsql = fmt.Sprintf("SELECT PRIVILEGE AS \"privileges\" FROM ROLE_SYS_PRIVS WHERE PRIVILEGE IN ('%s') GROUP BY PRIVILEGE", strings.Join(priAllTableS, "','")) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err @@ -282,11 +282,11 @@ func (or *QueryTable) TableAccessPriCheck(db *sql.DB, checkTableList []string, d var DM = make(map[string]int) for _, D := range checkTableList { DM[D]++ - if or.LowerCaseTableNames == "no" { + if or.CaseSensitiveObjectName == "no" { DM[strings.ToUpper(D)]++ } } - strsql = fmt.Sprintf("select owner||'.'||table_name AS \"tablesName\",PRIVILEGE as \"privileges\" from user_tab_privs") + strsql = fmt.Sprintf("SELECT owner||'.'||table_name AS \"tablesName\", PRIVILEGE AS \"privileges\" FROM user_tab_privs") if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { return nil, err } @@ -297,7 +297,7 @@ func (or *QueryTable) TableAccessPriCheck(db *sql.DB, checkTableList []string, d DD := columnMerge(tablePri, "tablesName", "privileges") for K, V := range DD { var aaaseq int - if or.LowerCaseTableNames == "no" { + if or.CaseSensitiveObjectName == "no" { strings.ToUpper(K) } if _, ok := DM[K]; ok { @@ -332,7 +332,7 @@ func (or *QueryTable) TableAllColumn(db *sql.DB, logThreadSeq int64) ([]map[stri ) vlog = fmt.Sprintf("(%d) [%s] Start to query the metadata of all the columns of table %s.%s in the %s database", logThreadSeq, Event, or.Schema, or.Table, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("SELECT column_name as \"columnName\",case when data_type='NUMBER' AND DATA_PRECISION is null THEN DATA_TYPE when data_type='NUMBER' AND DATA_PRECISION is not null then DATA_TYPE || '(' || DATA_PRECISION || ',' || NVL(DATA_SCALE,0) || ')' when data_type='VARCHAR2' THEN DATA_TYPE||'('||DATA_LENGTH||')' ELSE DATA_TYPE END AS \"dataType\",COLUMN_id as \"columnSeq\",NULLABLE as \"isNull\" FROM all_tab_columns WHERE owner='%s' and TABLE_NAME = '%s' order by column_id", or.Schema, or.Table) + strsql = fmt.Sprintf("SELECT column_name AS \"columnName\", CASE WHEN data_type='NUMBER' AND DATA_PRECISION IS NULL THEN DATA_TYPE WHEN data_type='NUMBER' AND DATA_PRECISION IS NOT NULL THEN DATA_TYPE || '(' || DATA_PRECISION || ',' || NVL(DATA_SCALE,0) || ')' WHEN data_type='VARCHAR2' THEN DATA_TYPE||'('||DATA_LENGTH||')' ELSE DATA_TYPE END AS \"dataType\", COLUMN_id AS \"columnSeq\", NULLABLE AS \"isNull\" FROM all_tab_columns WHERE owner='%s' AND TABLE_NAME='%s' ORDER BY column_id", or.Schema, or.Table) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} if dispos.SqlRows, err = dispos.DBSQLforExec(strsql); err != nil { @@ -493,7 +493,7 @@ func (or *QueryTable) TableIndexChoice(queryData []map[string]interface{}, logTh } //vlog = fmt.Sprintf("(%d) MySQL DB nounique key index starts to choose the best.", logThreadSeq) //global.Wlog.Debug(vlog) - f := or.keyChoiceDispos(multiseriateIndexColumnMap, "mui") + f := or.keyChoiceDispos(multiseriateIndexColumnMap, "mul") for k, v := range f { if len(v) > 0 { indexChoice[k] = v @@ -516,7 +516,7 @@ func (or *QueryTable) Trigger(db *sql.DB, logThreadSeq int64) (map[string]string ) vlog = fmt.Sprintf("(%d) [%s] Start to query the trigger information under the %s database.", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select TRIGGER_name as triggerName,TABLE_NAME as tableName from all_triggers where owner = '%s'", or.Schema) + strsql = fmt.Sprintf("SELECT TRIGGER_name AS triggerName, TABLE_NAME AS tableName FROM all_triggers WHERE owner='%s'", or.Schema) //vlog = fmt.Sprintf("(%d) Oracle DB query table query Trigger info exec sql is {%s}", logThreadSeq, sqlStr) //global.Wlog.Debug(vlog) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} @@ -582,7 +582,7 @@ func (or *QueryTable) Proc(db *sql.DB, logThreadSeq int64) (map[string]string, e ) vlog = fmt.Sprintf("(%d) [%s] Start to query the stored procedure information under the %s database.", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf(" select object_name as ROUTINE_NAME from all_procedures where object_type='PROCEDURE' and owner = '%s'", or.Schema) + strsql = fmt.Sprintf(" SELECT object_name AS ROUTINE_NAME FROM all_procedures WHERE object_type='PROCEDURE' AND owner='%s'", or.Schema) //vlog = fmt.Sprintf("(%d) Oracle DB query table query Stored Procedure info exec sql is {%s}", logThreadSeq, sqlStr) //global.Wlog.Debug(vlog) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} @@ -631,7 +631,7 @@ func (or *QueryTable) Func(db *sql.DB, logThreadSeq int64) (map[string]string, e ) vlog = fmt.Sprintf("(%d) [%s] Start to query the stored Func information under the %s database.", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf("select OBJECT_NAME as ROUTINE_NAME from all_procedures where object_type='FUNCTION' and owner = '%s'", or.Schema) + strsql = fmt.Sprintf("SELECT OBJECT_NAME AS ROUTINE_NAME FROM all_procedures WHERE object_type='FUNCTION' AND owner='%s'", or.Schema) //vlog = fmt.Sprintf("(%d) Oracle DB query table query Stored Function info exec sql is {%s}", logThreadSeq, sqlStr) //global.Wlog.Debug(vlog) dispos := dataDispos.DBdataDispos{DBType: DBType, LogThreadSeq: logThreadSeq, Event: Event, DB: db} @@ -680,7 +680,7 @@ func (or *QueryTable) Foreign(db *sql.DB, logThreadSeq int64) (map[string]string ) vlog = fmt.Sprintf("(%d) [%s] Start to query the Foreign information under the %s database.", logThreadSeq, Event, DBType) global.Wlog.Debug(vlog) - strsql = fmt.Sprintf(" select c.OWNER as DATABASE,c.table_name as TABLENAME, c.r_constraint_name,c.delete_rule,cc.column_name,cc.position from user_constraints c join user_cons_columns cc on c.constraint_name=cc.constraint_name and c.table_name=cc.table_name where c.constraint_type='R' and c.validated='VALIDATED' and c.OWNER = '%s' and c.table_name='%s'", or.Schema, or.Table) + strsql = fmt.Sprintf(" SELECT c.OWNER AS DATABASE, c.table_name AS TABLENAME, c.r_constraint_name, c.delete_rule, cc.column_name, cc.position FROM user_constraints c JOIN user_cons_columns cc ON c.constraint_name=cc.constraint_name AND c.table_name=cc.table_name WHERE c.constraint_type='R' AND c.validated='VALIDATED' AND c.OWNER = '%s' AND c.table_name='%s'", or.Schema, or.Table) //vlog = fmt.Sprintf("(%d) Oracle DB query table query Foreign info exec sql is {%s}", logThreadSeq, sqlStr) //global.Wlog.Debug(vlog) //sqlRows, err := db.Query(sqlStr) @@ -711,7 +711,7 @@ func (or *QueryTable) Foreign(db *sql.DB, logThreadSeq int64) (map[string]string } for k, _ := range routineNameM { schema, table := strings.Split(k, ".")[0], strings.Split(k, ".")[1] - strsql = fmt.Sprintf("SELECT DBMS_METADATA.GET_DDL('TABLE','%s','%s') as CREATE_FOREIGN FROM DUAL", table, schema) + strsql = fmt.Sprintf("SELECT DBMS_METADATA.GET_DDL('TABLE','%s','%s') AS CREATE_FOREIGN FROM DUAL", table, schema) //vlog = fmt.Sprintf("(%d) MySQL DB query create Foreign table %s.%s info, exec sql is {%s}", logThreadSeq, or.Schema, or.Table, sqlStr) //global.Wlog.Debug(vlog) //sqlRows, err = db.Query(sqlStr) diff --git a/README.md b/README.md index 394bcb8caabcba7d4b5fe9a70256249eb116f50f..4da5230ea5914027caeeb53fb5ac9eda59bb7d1c 100644 --- a/README.md +++ b/README.md @@ -1,183 +1,114 @@ [![](https://img.shields.io/badge/GreatSQL-官网-orange.svg)](https://greatsql.cn/) [![](https://img.shields.io/badge/GreatSQL-论坛-brightgreen.svg)](https://greatsql.cn/forum.php) [![](https://img.shields.io/badge/GreatSQL-博客-brightgreen.svg)](https://greatsql.cn/home.php?mod=space&uid=10&do=blog&view=me&from=space) -[![](https://img.shields.io/badge/License-Apache_v2.0-blue.svg)](https://gitee.com/GreatSQL/GreatSQL/blob/master/LICENSE) +[![](https://img.shields.io/badge/License-Apache_v2.0-blue.svg)](https://gitee.com/yejr/gt-checksum/blob/master/LICENSE) [![](https://img.shields.io/badge/release-1.2.1-blue.svg)](https://gitee.com/GreatSQL/gt-checksum/releases/tag/1.2.1) -# 关于 gt-checksum -gt-checksum是GreatSQL社区开源的一款静态数据库校验修复工具，支持MySQL、Oracle等主流数据库。 +# gt-checksum +**gt-checksum** 是GreatSQL社区开源的数据库校验及修复工具，支持MySQL、Oracle等主流数据库。 -# 特性 ---- -MySQL DBA最常用的数据校验&修复工具应该是Percona Toolkit中的pt-table-checksum和pt-table-sync这两个工具，不过这两个工具并不支持MySQL MGR架构，以及国内常见的上云下云业务场景，还有MySQL、Oracle间的异构数据库等多种场景。 +## 简介 -GreatSQL开源的gt-checksum工具可以满足上述多种业务需求场景，解决这些痛点。 +MySQL DBA经常使用 **pt-table-checksum** 和 **pt-table-sync** 进行数据校验及修复，但这两个工具并不支持MySQL MGR架构，以及国内常见的上云下云业务场景，还有MySQL、Oracle间的异构数据库等多种场景。 -gt-checksum工具支持以下几种常见业务需求场景： -1. **MySQL主从复制**：主从复制中断后较长时间才发现，且主从间差异的数据量太多，这时候通常基本上只能重建复制从库，如果利用pt-table-checksum先校验主从数据一致性后，再利用pt-table-sync工具修复差异数据，这个过程要特别久，时间代价太大。 -2. **MySQL MGR组复制**：MySQL MGR因故崩溃整个集群报错退出，或某个节点异常退出，在恢复MGR集群时一般要面临着先检查各节点间数据一致性的需求，这时通常为了省事会选择其中一个节点作为主节点，其余从节点直接复制数据重建，这个过程要特别久，时间代价大。 -3. **上云下云业务场景**：目前上云下云的业务需求很多，在这个过程中要进行大量的数据迁移及校验工作，如果出现字符集改变导致特殊数据出现乱码或其他的情况，如果数据迁移工具在迁移过程中出现bug或者数据异常而又迁移成功，此时都需要在迁移结束后进行一次数据校验才放心。 -4. **异构迁移场景**：有时我们会遇到异构数据迁移场景，例如从Oracle迁移到MySQL，通常存在字符集不同，以及数据类型不同等情况，也需要在迁移结束后进行一次数据校验才放心。 -5. **定期校验场景**：作为DBA在维护高可用架构中为了保证主节点出现异常后能够快速放心切换，就需要保证各节点间的数据一致性，需要定期执行数据校验工作。 +因此，我们开发了 **gt-checksum** 工具，旨在解决MySQL目标是支持更多业务需求场景，解决一些痛点。 -以上这些场景，都可以利用gt-chcksum工具来满足。 +**gt-checksum** 支持以下几种常见业务需求场景： +1. **MySQL主从复制**：当主从复制中断较长时间后才发现，主从间数据差异太大。此时通常选择重建整个从库，如果利用 **pt-table-checksum**、**pt-table-sync** 先校验后修复，这个过程通常特别久，时间代价太大。而 **gt-checksum** 工作效率更高，可以更快校验出主从间数据差异并修复，这个过程时间代价小很多。 +2. **MySQL MGR组复制**：MySQL MGR因故报错运行异常或某个节点异常退出时，在恢复时一般要先检查各节点间数据一致性，这时通常选择其中一个节点作为主节点，其余从节点直接复制数据重建，整个过程要特别久，时间代价大。在这种场景下选择使用 **gt-checksum** 效率更高。 +3. **企业上下云**：在企业上云下云过程中要进行大量的数据迁移及校验工作，可能存在字符集原因导致个别数据出现乱码或其他情况，在迁移结束后进行完整的数据校验就很有必要了。 +4. **异构迁移**：例如从Oracle迁移到MySQL等异构数据库迁移场景中，通常存在字符集不同、数据类型不同等多种复杂情况，也需要在迁移结束后进行完整的数据校验。 +5. **定期数据校验**：在多节点高可用架构中，为了保证主节点出现异常后能安心切换，需要确保各节点间的数据一致性，通常要定期执行数据校验工作。 -# 下载 ---- -可以 [这里](https://gitee.com/GreatSQL/gt-checksum/releases) 下载预编译好的二进制文件包，已经在Ubuntu、CentOS、RHEL等多个下测试通过。 +## 下载 -如果需要校验Oracle数据库，则还需要先下载Oracle数据库相应版本的驱动程序，并配置驱动程序使之生效。例如：待校验的数据库为Oracle 11-2，则要下载Oracle 11-2的驱动程序，并使之生效，否则连接Oracle会报错。详细方法请见下方内容：[**下载配置Oracle驱动程序**](#%E4%B8%8B%E8%BD%BD%E9%85%8D%E7%BD%AEoracle%E9%A9%B1%E5%8A%A8%E7%A8%8B%E5%BA%8F)。 +可以 [这里](https://gitee.com/GreatSQL/gt-checksum/releases) 下载预编译好的二进制文件包，已经在 Ubuntu、CentOS、RHEL 等多个系统环境下测试通过。 -# 快速运行 ---- -```shell -# 不带任何参数 -shell> ./gt-checksum -If no parameters are loaded, view the command with --help or -h +如果需要校验Oracle数据库，则还需要先下载Oracle数据库相应版本的驱动程序，并配置驱动程序使之生效。例如：待校验的数据库为Oracle 11-2，则要下载Oracle 11-2的驱动程序，并使之生效，否则连接Oracle会报错。详细方法请见下方内容：[**下载配置Oracle驱动程序**](./gt-checksum-manual.md#下载配置Oracle驱动程序)。 -# 查看版本号 -shell> ./gt-checksum -v -gt-checksum version 1.2.1 - -# 查看使用帮助 -shell> ./gt-checksum -h -NAME: - gt-checksum - A opensource table and data checksum tool by GreatSQL +## 快速运行 -USAGE: - gt-checksum [global options] command [command options] [arguments...] -... - -# 数据库授权 -# 想要运行gt-checksum工具，需要至少授予以下几个权限 -# MySQL端 -# 1.全局权限 -# a.`REPLICATION CLIENT` -# b.`SESSION_VARIABLES_ADMIN`，如果是MySQL 8.0版本的话，MySQL 5.7版本不做这个要求 -# 2.校验数据对象 -# a.如果`datafix=file`，则只需要`SELECT`权限 -# b.如果`datafix=table`，则需要`SELECT、INSERT、DELETE`权限，如果还需要修复表结构不一致的情况，则需要`ALTER`权限 -# -# 假设现在要对db1.t1做校验和修复，则可授权如下 - -mysql> GRANT REPLICATION CLIENT, SESSION_VARIABLES_ADMIN ON *.* to ...; -mysql> GRANT SELECT, INSERT, DELETE ON db1.t1 to ...; - -# Oracle端 -# 1.全局权限 -# a.`SELECT ANY DICTIONARY` -# 2.校验数据对象 -# a.如果`datafix=file`，则只需要`SELECT ANY TABLE`权限 -# b.如果`datafix=table`，则需要`SELECT ANY TABLE、INSERT ANY TABLE、DELETE ANY TABLE`权限 - -# 指定配置文件，开始执行数据校验，示例： -shell> ./gt-checksum -f ./gc.conf --- gt-checksum init configuration files -- --- gt-checksum init log files -- --- gt-checksum init check parameter -- --- gt-checksum init check table name -- --- gt-checksum init check table column -- --- gt-checksum init check table index column -- --- gt-checksum init source and dest transaction snapshoot conn pool -- --- gt-checksum init cehck table query plan and check data -- -begin checkSum index table db1.t1 -[█████████████████████████████████████████████████████████████████████████████████████████████████████████████████]113% task: 678/600 -table db1.t1 checksum complete +- 不带任何参数 -** gt-checksum Overview of results ** -Check time: 73.81s (Seconds) -Schema Table IndexCol checkMod Rows Differences Datafix -db1 t1 ol_w_id,ol_d_id,ol_o_id,ol_number rows 5995934,5995918 yes file - - -# 使用命令行传参方式执行数据校验 -shell> ./gt-checksum -S type=mysql,user=checksum,passwd=Checksum@123,host=172.16.0.1,port=3306,charset=utf8 -D type=mysql,user=checksum,passwd=Checksum@123,host=172.16.0.2,port=3306,charset=utf8 -t test.t2 -nit yes --- gt-checksum init configuration files -- --- gt-checksum init log files -- --- gt-checksum init check parameter -- --- gt-checksum init check table name -- --- gt-checksum init check table column -- --- gt-checksum init check table index column -- --- gt-checksum init source and dest transaction snapshoot conn pool -- --- gt-checksum init cehck table query plan and check data -- -begin checkSum index table SCOTT.A5 -[█ ]100% task: 1/1 -table SCOTT.A5 checksum complete - -** gt-checksum Overview of results ** -Check time: 0.29s (Seconds) -Schema Table IndexCol checkMod Rows Differences Datafix -test t2 id rows 10,10 no file +```bash +$ gt-checksum +No config file specified and there is no gc.conf in the current directory, run the command with -h or --help ``` -# 下载配置Oracle驱动程序 ---- -如果需要校验Oracle数据库，则还需要先下载Oracle数据库相应版本的驱动程序。例如：待校验的数据库为Oracle 11-2，则要下载Oracle 11-2的驱动程序，并使之生效，否则连接Oracle会报错。 +如果当前目录下有配置文件*gc.conf*，则会读取该配置文件开始运行，例如： -## 下载Oracle Instant Client -从 [https://www.oracle.com/database/technologies/instant-client/downloads.html](https://www.oracle.com/database/technologies/instant-client/downloads.html) 下载免费的Basic或Basic Light软件包。 +```bash +$ gt-checksum -- oracle basic client, instantclient-basic-linux.x64-11.2.0.4.0.zip +gt-checksum: Automatically loading configuration file 'gc.conf' from current directory. -- oracle sqlplus, instantclient-sqlplus-linux.x64-11.2.0.4.0.zip +gt-checksum is initializing +gt-checksum is reading configuration files +``` -- oracle sdk, instantclient-sdk-linux.x64-11.2.0.4.0.zip +- 查看版本号 -## 配置oracle client并生效 -```shell -shell> unzip instantclient-basic-linux.x64-11.2.0.4.0.zip -shell> unzip instantclient-sqlplus-linux.x64-11.2.0.4.0.zip -shell> unzip instantclient-sdk-linux.x64-11.2.0.4.0.zip -shell> mv instantclient_11_2 /usr/local -shell> echo "export LD_LIBRARY_PATH=/usr/local/instantclient_11_2:$LD_LIBRARY_PATH" >> /etc/profile -shell> source /etc/profile +```bash +$ gt-checksum -v +gt-checksum version 1.2.1 ``` -# 源码编译 -gt-checksum工具采用GO语言开发，您可以自行编译生成二进制文件。 +- 查看使用帮助 -编译环境要求使用golang 1.17及以上版本。 +```bash +$ gt-checksum -h +NAME: + gt-checksum - opensource database checksum and sync tool by GreatSQL -请参考下面方法下载源码并进行编译： -```shell -shell> git clone https://gitee.com/GreatSQL/gt-checksum.git -shell> go build -o gt-checksum gt-checksum.go -shell> chmod +x gt-checksum -shell> mv gt-checksum /usr/local/bin +USAGE: + gt-checksum [global options] command [command options] [arguments...] ``` -也可以直接利用Docker环境编译，在已经准备好Docker运行环境的基础上，执行如下操作即可： -```shell -shell> git clone https://gitee.com/GreatSQL/gt-checksum.git -shell> cd gt-checksum -shell> DOCKER_BUILDKIT=1 docker build --build-arg VERSION=v1.2.1 -f Dockerfile -o ./ . -shell> cd gt-checksum-v1.2.1 -shell> ./gt-checksum -v -gt-checksum version 1.2.1 +- 指定配置文件方式，执行数据校验 + +拷贝或重命名模板文件*gc-sample.conf*为*gc.conf*，主要修改`srcDSN`,`dstDSN`,`tables`,`ignoreTables`等几个参数后，执行如下命令进行数据校验： + +```bash +$ gt-checksum -c ./gc.conf + +gt-checksum is initializing +gt-checksum is reading configuration files +gt-checksum is opening log files +gt-checksum is checking options +gt-checksum is opening check tables +gt-checksum is opening table columns +gt-checksum is opening table indexes +gt-checksum is opening srcDSN and dstDSN +gt-checksum is generating tables and data check plan +begin checkSum index table db1.t1 +[█████████████████████████████████████████████████████████████████████████████████████████████████████████████████]113% task: 678/600 +table db1.t1 checksum complete + +** gt-checksum Overview of results ** +Check time: 73.81s (Seconds) +Schema Table IndexColumn checkMode Rows Diffs Datafix +db1 t1 ol_w_id,ol_d_id,ol_o_id,ol_number rows 5995934,5995918 yes file ``` -这就编译完成并可以开始愉快地玩耍了。 -# 使用文档 ---- -- [gt-checksum manual](https://gitee.com/GreatSQL/gt-checksum/blob/master/docs/gt-checksum-manual.md) +> 开始执行数据校验前，要先在源和目标数据库创建相应的专属账号并授权。详情参考：[**gt-checksum 手册**](./gt-checksum-manual.md#数据库授权)。 + +## 手册 + +[gt-checksum 手册](./gt-checksum-manual.md) +## 版本历史 -# 版本历史 ---- -- [版本历史](https://gitee.com/GreatSQL/gt-checksum/blob/master/relnotes/CHANGELOG.zh-CN.md) +[版本历史](./CHANGELOG.zh-CN.md) +## 配置参数 -# 已知缺陷 ---- -截止最新的1.2.1版本中，当表中有多行数据是完全重复的话，可能会导致校验结果不准确，详见 [已知缺陷](https://gitee.com/GreatSQL/gt-checksum/blob/master/docs/gt-checksum-manual.md#已知缺陷) 。 +配置文件中所有参数的详解可参考模板文件 [gc-sample.conf](./gc-sample.conf)。 -# 问题反馈 ---- -- [问题反馈 gitee](https://gitee.com/GreatSQL/gt-checksum/issues) +## 问题反馈 +可以 [提交issue](https://gitee.com/GreatSQL/gt-checksum/issues) 查看或提交 gt-checksum 相关bug。 -# 联系我们 ---- +## 联系我们 扫码关注微信公众号 diff --git a/actions/TerminalResultOutput.go b/actions/TerminalResultOutput.go index 1c4385a9c20682390ed8eacab31e1c170a0162ca..298c2a8a3bd6dee02148b6b563095ce05092234c 100644 --- a/actions/TerminalResultOutput.go +++ b/actions/TerminalResultOutput.go @@ -6,20 +6,24 @@ import ( "github.com/gosuri/uitable" "gt-checksum/inputArg" "strings" + "time" ) //进度条 type Bar struct { - percent int64 //百分比 - cur int64 //当前进度位置 - total int64 //总进度 - rate string //进度条 - graph string //显示符号 - taskUnit string //task单位 + percent int64 //百分比 + cur int64 //当前进度位置 + total int64 //总进度 + rate string //进度条 + graph string //显示符号 + taskUnit string //task单位 + lastUpdate int64 //上次更新时间戳（毫秒） + updateInterval int64 //更新间隔（毫秒） + startTime int64 //开始时间戳（毫秒） } type Pod struct { - Schema, Table, IndexCol, CheckMod, Rows, Differences, CheckObject, Datafix, FuncName, Definer, ProcName, Sample, TriggerName string + Schema, Table, IndexColumn, CheckMode, Rows, DIFFS, CheckObject, Datafix, FuncName, Definer, ProcName, Sample, TriggerName string } var measuredDataPods []Pod @@ -32,70 +36,70 @@ func CheckResultOut(m *inputArg.ConfigParameter) { switch m.SecondaryL.RulesV.CheckObject { case "struct": - table.AddRow("Schema", "Table ", " CheckObject ", "Differences", "Datafix") + table.AddRow("Schema", "Table", " CheckObject ", "Diffs", "Datafix") for _, pod := range measuredDataPods { - table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.CheckObject), color.GreenString(pod.Differences), color.YellowString(pod.Datafix)) + table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.CheckObject), color.GreenString(pod.DIFFS), color.YellowString(pod.Datafix)) } fmt.Println(table) case "index": - table.AddRow("Schema", "Table ", "CheckObject ", "Differences", "Datafix") + table.AddRow("Schema", "Table", "CheckObject ", "Diffs", "Datafix") for _, pod := range measuredDataPods { - table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.CheckObject), color.GreenString(pod.Differences), color.YellowString(pod.Datafix)) + table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.CheckObject), color.GreenString(pod.DIFFS), color.YellowString(pod.Datafix)) } fmt.Println(table) case "partitions": - table.AddRow("Schema", "Table ", "checkObject ", "Differences", "Datafix") + table.AddRow("Schema", "Table", "checkObject ", "Diffs", "Datafix") for _, pod := range measuredDataPods { - table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.CheckObject), color.GreenString(pod.Differences), color.YellowString(pod.Datafix)) + table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.CheckObject), color.GreenString(pod.DIFFS), color.YellowString(pod.Datafix)) } fmt.Println(table) case "foreign": - table.AddRow("Schema", "Table ", "checkObject ", "Differences", "Datafix") + table.AddRow("Schema", "Table", "checkObject ", "Diffs", "Datafix") for _, pod := range measuredDataPods { - table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.CheckObject), color.GreenString(pod.Differences), color.YellowString(pod.Datafix)) + table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.CheckObject), color.GreenString(pod.DIFFS), color.YellowString(pod.Datafix)) } fmt.Println(table) case "func": - table.AddRow("Schema ", "funcName ", "checkObject ", "Differences ", "Datafix ") + table.AddRow("Schema ", "funcName ", "checkObject ", "DIFFS ", "Datafix ") for _, pod := range measuredDataPods { - table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.FuncName), color.RedString(pod.CheckObject), color.GreenString(pod.Differences), color.YellowString(pod.Datafix)) + table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.FuncName), color.RedString(pod.CheckObject), color.GreenString(pod.DIFFS), color.YellowString(pod.Datafix)) } fmt.Println(table) case "proc": - table.AddRow("Schema ", "procName ", "checkObject ", "Differences ", "Datafix ") + table.AddRow("Schema ", "procName ", "checkObject ", "DIFFS ", "Datafix ") for _, pod := range measuredDataPods { - table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.ProcName), color.RedString(pod.CheckObject), color.GreenString(pod.Differences), color.YellowString(pod.Datafix)) + table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.ProcName), color.RedString(pod.CheckObject), color.GreenString(pod.DIFFS), color.YellowString(pod.Datafix)) } fmt.Println(table) case "trigger": - table.AddRow("Schema ", "triggerName ", "checkObject ", "Differences ", "Datafix ") + table.AddRow("Schema ", "triggerName ", "checkObject ", "Diffs ", "Datafix ") for _, pod := range measuredDataPods { - table.AddRow(color.RedString(pod.Schema), color.GreenString(pod.TriggerName), color.RedString(pod.CheckObject), color.GreenString(pod.Differences), color.YellowString(pod.Datafix)) + table.AddRow(color.RedString(pod.Schema), color.GreenString(pod.TriggerName), color.RedString(pod.CheckObject), color.GreenString(pod.DIFFS), color.YellowString(pod.Datafix)) } fmt.Println(table) case "data": switch m.SecondaryL.RulesV.CheckMode { case "count": - table.AddRow("Schema", "Table ", "checkObject", "checkMod", "Rows", "Differences") + table.AddRow("Schema", "Table", "checkObject", "checkMode", "Rows", "Diffs") for _, pod := range measuredDataPods { - table.AddRow(color.RedString(pod.Schema), color.GreenString(pod.Table), color.RedString(pod.CheckObject), color.GreenString(pod.CheckMod), color.RedString(pod.Rows), color.YellowString(pod.Differences)) + table.AddRow(color.RedString(pod.Schema), color.GreenString(pod.Table), color.RedString(pod.CheckObject), color.GreenString(pod.CheckMode), color.RedString(pod.Rows), color.YellowString(pod.DIFFS)) } fmt.Println(table) case "sample": for _, pod := range measuredDataPods { if pod.Sample == "" { - table.AddRow("Schema", "Table ", "IndexCol ", "checkObject", "checkMod", "Rows", "Differences") - table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.IndexCol), color.YellowString(pod.CheckObject), color.BlueString(pod.CheckMod), color.BlueString(pod.Rows), color.GreenString(pod.Differences)) + table.AddRow("Schema", "Table", "IndexColumn", "checkObject", "checkMode", "Rows", "Diffs") + table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.IndexColumn), color.YellowString(pod.CheckObject), color.BlueString(pod.CheckMode), color.BlueString(pod.Rows), color.GreenString(pod.DIFFS)) } else { - table.AddRow("Schema", "Table ", "IndexCol ", "checkObject", "checkMod", "Rows", "Samp", "Differences") - table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.IndexCol), color.YellowString(pod.CheckObject), color.BlueString(pod.CheckMod), color.BlueString(pod.Rows), color.RedString(pod.Sample), color.GreenString(pod.Differences)) + table.AddRow("Schema", "Table", "IndexColumn", "checkObject", "checkMode", "Rows", "Samp", "Diffs") + table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.IndexColumn), color.YellowString(pod.CheckObject), color.BlueString(pod.CheckMode), color.BlueString(pod.Rows), color.RedString(pod.Sample), color.GreenString(pod.DIFFS)) } } fmt.Println(table) case "rows": - table.AddRow("Schema", "Table ", "IndexCol ", "checkMod", "Rows", "Differences", "Datafix") + table.AddRow("Schema", "Table", "IndexColumn", "checkMode", "Rows", "Diffs", "Datafix") for _, pod := range measuredDataPods { - var differences = pod.Differences + var differences = pod.DIFFS for k, _ := range differencesSchemaTable { if k != "" { KI := strings.Split(k, "greatdbCheck_greatdbCheck") @@ -104,47 +108,10 @@ func CheckResultOut(m *inputArg.ConfigParameter) { } } } - table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.IndexCol), color.BlueString(pod.CheckMod), color.BlueString(pod.Rows), color.GreenString(differences), color.YellowString(pod.Datafix)) + table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.IndexColumn), color.BlueString(pod.CheckMode), color.BlueString(pod.Rows), color.GreenString(differences), color.YellowString(pod.Datafix)) } fmt.Println(table) } - //if m.CheckObject == "data" { - //if m.CheckMode == "count" { - // table.AddRow("Schema", "Table ", "checkObject", "checkMod", "Rows", "Differences") - // for _, pod := range measuredDataPods { - // table.AddRow(color.RedString(pod.Schema), color.GreenString(pod.Table), color.RedString(pod.CheckObject), color.GreenString(pod.CheckMod), color.RedString(pod.Rows), color.YellowString(pod.Differences)) - // } - // fmt.Println(table) - //} - //if m.CheckMode == "sample" { - // for _, pod := range measuredDataPods { - // if pod.Sample == "" { - // table.AddRow("Schema", "Table ", "IndexCol ", "checkObject", "checkMod", "Rows", "Differences") - // table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.IndexCol), color.YellowString(pod.CheckObject), color.BlueString(pod.CheckMod), color.BlueString(pod.Rows), color.GreenString(pod.Differences)) - // } else { - // table.AddRow("Schema", "Table ", "IndexCol ", "checkObject", "checkMod", "Rows", "Samp", "Differences") - // table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.IndexCol), color.YellowString(pod.CheckObject), color.BlueString(pod.CheckMod), color.BlueString(pod.Rows), color.RedString(pod.Sample), color.GreenString(pod.Differences)) - // } - // } - // fmt.Println(table) - //} - //if m.CheckMode == "rows" { - // table.AddRow("Schema", "Table ", "IndexCol ", "checkMod", "Rows", "Differences", "Datafix") - // for _, pod := range measuredDataPods { - // var differences = pod.Differences - // for k, _ := range differencesSchemaTable { - // if k != "" { - // KI := strings.Split(k, "greatdbCheck_greatdbCheck") - // if pod.Schema == KI[0] && pod.Table == KI[1] { - // differences = "yes" - // } - // } - // } - // table.AddRow(color.RedString(pod.Schema), color.WhiteString(pod.Table), color.RedString(pod.IndexCol), color.BlueString(pod.CheckMod), color.BlueString(pod.Rows), color.GreenString(differences), color.YellowString(pod.Datafix)) - // } - // fmt.Println(table) - //} - //} } } @@ -152,17 +119,27 @@ func (bar *Bar) NewOption(start, total int64, taskUnit string) { bar.cur = start bar.total = total bar.taskUnit = taskUnit + bar.updateInterval = 100 // 调整为100毫秒更新一次，使进度条更流畅 + bar.startTime = time.Now().UnixMilli() // 记录开始时间 if bar.graph == "" { bar.graph = "█" } bar.percent = bar.getPercent() - for i := 0; i < int(bar.percent); i += 2 { - bar.rate += bar.graph //初始化进度条位置 - } + // 计算进度条长度：每个█字符代表5%的进度（100% / 20个字符） + progressBars := int(float64(bar.percent) * 20 / 100) + bar.rate = strings.Repeat(bar.graph, progressBars) //初始化进度条位置 } func (bar *Bar) getPercent() int64 { - return int64(float32(bar.cur) / float32(bar.total) * 100) + if bar.total == 0 { + return 0 + } + percent := int64(float32(bar.cur) / float32(bar.total) * 100) + // 确保百分比不超过100% + if percent > 100 { + return 100 + } + return percent } func (bar *Bar) NewOptionWithGraph(start, total int64, graph, taskUnit string) { bar.graph = graph @@ -176,23 +153,53 @@ func (bar *Bar) Play(cur int64) { bar.cur = cur last := bar.percent bar.percent = bar.getPercent() - //if bar.percent != last && bar.percent%2 == 0 { - // bar.rate += bar.graph - //} - if bar.percent != last { - bar.rate += bar.graph + + currentTime := time.Now().UnixMilli() + + // 强制在进度完成时更新进度条 + if bar.percent == 100 || bar.cur == bar.total { + // 补全进度条到100% (20个█字符) + for len(bar.rate) < 20 { + bar.rate += bar.graph + } + bar.percent = 100 + // 计算实时耗时（秒） + elapsedMilliseconds := time.Now().UnixMilli() - bar.startTime + fmt.Printf("\r\033[K[%-20s]%3d%% %s%5d/100 Elapsed time: %.2fs", bar.rate, bar.percent, fmt.Sprintf("%s:", bar.taskUnit), bar.percent, float64(elapsedMilliseconds)/1000) + } else if (bar.percent != last || bar.cur == bar.total) && (currentTime - bar.lastUpdate) >= bar.updateInterval { + // 只在百分比变化且达到更新时间间隔时才更新进度条 + // 计算当前应该显示的进度条长度（每个█字符代表5%的进度） + progressBars := int(float64(bar.percent) * 20 / 100) + // 确保进度条长度不超过20个字符 + if progressBars > 20 { + progressBars = 20 + } + bar.rate = strings.Repeat(bar.graph, progressBars) + bar.lastUpdate = currentTime + // 使用回车符覆盖当前行，避免刷屏 + // 计算实时耗时（秒） + elapsedMilliseconds := currentTime - bar.startTime + fmt.Printf("\r\033[K[%-20s]%3d%% %s%5d/100 Elapsed: %.2fs", bar.rate, bar.percent, fmt.Sprintf("%s:", bar.taskUnit), bar.percent, float64(elapsedMilliseconds)/1000) } - //if bar.total >= 100 - //if bar.taskUnit == "task"{ - // fmt.Printf("\r[%-21s]%3d%% %s%8d/%d", bar.rate, bar.percent, "task:", bar.cur, bar.total) - //} - //if bar.taskUnit == "rows"{ - // fmt.Printf("\r[%-21s]%3d%% %s%8d/%d", bar.rate, bar.percent, "rows:", bar.cur, bar.total) - //} - fmt.Printf("\r[%-21s]%3d%% %s%8d/%d", bar.rate, bar.percent, fmt.Sprintf("%s:", bar.taskUnit), bar.cur, bar.total) +} + +// NewTableProgress 开始新表的进度显示，先输出换行再开始进度条 +func (bar *Bar) NewTableProgress(tableName string) { + // 先输出换行确保新表进度在新行开始 + fmt.Printf("\n%-40s", tableName) } //由于上面的打印没有打印换行符，因此，在进度全部结束之后（也就是跳出循环之外时），需要打印一个换行符，因此，封装了一个Finish函数，该函数纯粹的打印一个换行，表示进度条已经完成。 func (bar *Bar) Finish() { + // 强制设置进度为100%并补全进度条 + bar.cur = bar.total + bar.percent = 100 + bar.rate = strings.Repeat(bar.graph, 20) // 强制补全进度条到20个字符 + + // 计算耗时（秒） + endTime := time.Now().UnixMilli() + elapsedSeconds := float64(endTime - bar.startTime) / 1000.0 + + fmt.Printf("\r\033[K[%-20s]%3d%% %s%5d/100 Elapsed: %.2fs", bar.rate, bar.percent, fmt.Sprintf("%s:", bar.taskUnit), bar.percent, elapsedSeconds) fmt.Println() } diff --git a/actions/differencesDataDispos.go b/actions/differencesDataDispos.go index 82524add60318befeac72a505cffa1a0dbb58795..8968dbc7a396547f0190a8c2050491c1be07082f 100644 --- a/actions/differencesDataDispos.go +++ b/actions/differencesDataDispos.go @@ -11,20 +11,20 @@ var rollbackSQL = func(sl []string) []string { var newDelS []string for _, i := range sl { if strings.HasPrefix(i, "insert") { - ii := strings.Replace(strings.Replace(i, "insert into", "delete from", 1), "values", "where", 1) + ii := strings.Replace(strings.Replace(i, "INSERT INTO", "DELETE FROM", 1), "VALUES", "WHERE", 1) newDelS = append(newDelS, ii) } if strings.HasPrefix(i, "update") { - schemaTable := strings.TrimSpace(strings.Split(strings.Split(i, "where")[0], "update")[1]) - e := strings.Split(strings.Split(i, "where")[1], "/*columnModify*/") + schemaTable := strings.TrimSpace(strings.Split(strings.Split(i, "WHERE")[0], "UPDATE")[1]) + e := strings.Split(strings.Split(i, "WHERE")[1], "/*columnModify*/") oldrow := strings.Replace(e[0], "(", "", 1) newrow := strings.Replace(e[1], ");", "", 1) - delSql := fmt.Sprintf("delete from %s where %s;", schemaTable, newrow) - addSql := fmt.Sprintf("insert into %s values (%s);", schemaTable, oldrow) + delSql := fmt.Sprintf("DELETE FROM %s WHERE %s;", schemaTable, newrow) + addSql := fmt.Sprintf("INSERT INTO %s VALUES (%s);", schemaTable, oldrow) newDelS = append(newDelS, delSql, addSql) } if strings.HasPrefix(i, "delete") { - ii := strings.Replace(strings.Replace(i, "delete from", "insert into", 1), "where", "values", 1) + ii := strings.Replace(strings.Replace(i, "DELETE FROM", "INSERT INTO", 1), "WHERE", "VALUES", 1) newDelS = append(newDelS, ii) } } @@ -35,18 +35,18 @@ var rollbackSQL = func(sl []string) []string { var positiveSequenceSQL = func(sl []string) []string { var newDelS []string for _, i := range sl { - if i != "" && strings.HasPrefix(i, "insert into") { + if i != "" && strings.HasPrefix(i, "INSERT INTO") { newDelS = append(newDelS, i) } - if i != "" && strings.HasPrefix(i, "delete") { + if i != "" && strings.HasPrefix(i, "DELETE") { newDelS = append(newDelS, i) } - if i != "" && strings.HasPrefix(i, "update") { - schemaTable := strings.TrimSpace(strings.Split(strings.Split(i, "where")[0], "update")[1]) + if i != "" && strings.HasPrefix(i, "UPDATE") { + schemaTable := strings.TrimSpace(strings.Split(strings.Split(i, "WHERE")[0], "UPDATE")[1]) e := strings.Split(i, "/*columnModify*/") - delSql := fmt.Sprintf("delete from %s);", strings.Replace(e[0], "update ", "", 1)) + delSql := fmt.Sprintf("DELETE FROM %s);", strings.Replace(e[0], "UPDATE ", "", 1)) newDelS = append(newDelS, delSql) - addSql := fmt.Sprintf("insert into %s values (%s", schemaTable, e[1]) + addSql := fmt.Sprintf("INSERT INTO %s VALUES (%s", schemaTable, e[1]) newDelS = append(newDelS, addSql) } } diff --git a/actions/incDataDispos.go b/actions/incDataDispos.go index 0432a04f9a6be87ac7b4cb0b4450810308e1365c..af3e7828fd96a3d6c51ae3f8addb512db615f3fd 100644 --- a/actions/incDataDispos.go +++ b/actions/incDataDispos.go @@ -177,7 +177,7 @@ func (idds IncDataDisposStruct) Aa(fullDataCompletionStatus chan struct{}, cqMq //读取源目端binlog的线程停止 if ok && ok1 { - fmt.Println("---退出__-") + fmt.Println("Exit!") break } } diff --git a/actions/rapirDML.go b/actions/rapirDML.go index 1953dce397d64a3e4536f2745cea74962541eb62..58e5a33190b5846e66ec7e91520fc394becf86f1 100644 --- a/actions/rapirDML.go +++ b/actions/rapirDML.go @@ -100,20 +100,20 @@ func (rs rapirSqlStruct) SqlFile(sfile *os.File, sql []string, logThreadSeq int6 sqlCommit []string ) vlog = fmt.Sprintf("(%d) Start writing repair statements to the repair file.", logThreadSeq) - global.Wlog.Info(vlog) + global.Wlog.Debug(vlog) if strings.HasPrefix(strings.ToUpper(strings.Join(sql, ";")), "ALTER TABLE") { sqlCommit = sql } else { - sqlCommit = []string{"begin;"} + sqlCommit = []string{"BEGIN;"} sqlCommit = append(sqlCommit, sql...) - sqlCommit = append(sqlCommit, "commit;") + sqlCommit = append(sqlCommit, "COMMIT;") } _, err := FileOperate{File: sfile, BufSize: 1024 * 4 * 1024, SqlType: "sql"}.ConcurrencyWriteFile(sqlCommit) if err != nil { return err } vlog = fmt.Sprintf("(%d) Write the repair statement to the repair file successfully.", logThreadSeq) - global.Wlog.Info(vlog) + global.Wlog.Debug(vlog) return nil } func ApplyDataFix(fixSql []string, datafixType string, sfile *os.File, ddrive, jdbc string, logThreadSeq int64) error { diff --git a/actions/schema_tab_struct.go b/actions/schema_tab_struct.go index 2661f890b3c193c654a5999619b18a4b7766f528..440b07fa96e5bdf8800aa537eb723d6b2bc68938 100644 --- a/actions/schema_tab_struct.go +++ b/actions/schema_tab_struct.go @@ -22,7 +22,7 @@ type schemaTable struct { destDrive string sourceDB *sql.DB destDB *sql.DB - lowerCaseTableNames string + caseSensitiveObjectName string datefix string sfile *os.File djdbc string @@ -57,7 +57,7 @@ func (stcls *schemaTable) tableColumnName(db *sql.DB, tc dbExec.TableColumnNameS if queryData, err = tc.Query().TableColumnName(db, logThreadSeq2); err != nil { return col, err } - vlog = fmt.Sprintf("(%d) [%s] start dispos DB query columns data. to dispos it...", logThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] start checking columns", logThreadSeq, Event) global.Wlog.Debug(vlog) for _, v := range queryData { if fmt.Sprintf("%v", v["columnName"]) != "" { @@ -68,7 +68,7 @@ func (stcls *schemaTable) tableColumnName(db *sql.DB, tc dbExec.TableColumnNameS for _, v := range CS { col = append(col, map[string][]string{v: A[v]}) } - vlog = fmt.Sprintf("(%d) [%s] complete dispos DB query columns data.", logThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] columns checksum completed", logThreadSeq, Event) global.Wlog.Debug(vlog) return col, nil } @@ -85,10 +85,10 @@ func (stcls *schemaTable) TableColumnNameCheck(checkTableList []string, logThrea tableAbnormalBool = false event string ) - vlog = fmt.Sprintf("(%d) %s Start to check the consistency information of source and target table structure and column information ...", logThreadSeq, event) + vlog = fmt.Sprintf("(%d) %s Start checking the differences between the table structure and columns of srcDSN and dstDSN", logThreadSeq, event) global.Wlog.Debug(vlog) for _, v := range checkTableList { - vlog = fmt.Sprintf("(%d %s Start to check the table structure consistency of table %s.", logThreadSeq, event, v) + vlog = fmt.Sprintf("(%d %s Start checking structure of table %s", logThreadSeq, event, v) global.Wlog.Debug(vlog) var sColumn, dColumn []map[string][]string stcls.schema = strings.Split(v, ".")[0] @@ -97,20 +97,20 @@ func (stcls *schemaTable) TableColumnNameCheck(checkTableList []string, logThrea tc := dbExec.TableColumnNameStruct{Schema: stcls.schema, Table: stcls.table, Drive: stcls.sourceDrive} sColumn, err = stcls.tableColumnName(stcls.sourceDB, tc, logThreadSeq, logThreadSeq2) if err != nil { - vlog = fmt.Sprintf("(%d) %s Querying the metadata information of table %s.%s in the source %s database failed, and the error message is {%s}", logThreadSeq, event, stcls.schema, stcls.table, stcls.sourceDrive, err) + vlog = fmt.Sprintf("(%d) %s Obtain metadata of table %s.%s in srcDSN {%s} failed: {%s}", logThreadSeq, event, stcls.schema, stcls.table, stcls.sourceDrive, err) global.Wlog.Error(vlog) return nil, nil, err } - vlog = fmt.Sprintf("(%d) %s source DB %s table name [%s.%s] column name message is {%v} num [%d]", logThreadSeq, event, stcls.sourceDrive, stcls.schema, stcls.table, sColumn, len(sColumn)) + vlog = fmt.Sprintf("(%d) %s srcDSN {%s} table: [%s.%s] [%d] columns: {%v}", logThreadSeq, event, stcls.sourceDrive, stcls.schema, stcls.table, len(sColumn), sColumn) global.Wlog.Debug(vlog) tc.Drive = stcls.destDrive dColumn, err = stcls.tableColumnName(stcls.destDB, tc, logThreadSeq, logThreadSeq2) if err != nil { - vlog = fmt.Sprintf("(%d) %s Querying the metadata information of table %s.%s in the source %s database failed, and the error message is {%s}", logThreadSeq, event, stcls.schema, stcls.table, stcls.destDrive, err) + vlog = fmt.Sprintf("(%d) %s Obtain metadata of table %s.%s in dstDSN {%s} failed: {%s}", logThreadSeq, event, stcls.schema, stcls.table, stcls.destDrive, err) global.Wlog.Error(vlog) return nil, nil, err } - vlog = fmt.Sprintf("(%d) %s dest DB %s table name [%s.%s] column name message is {%v} num [%d]", logThreadSeq, event, stcls.destDrive, stcls.schema, stcls.table, dColumn, len(dColumn)) + vlog = fmt.Sprintf("(%d) %s dstDSN {%s} table: [%s.%s] [%d] columns: {%v}", logThreadSeq, event, stcls.destDrive, stcls.schema, stcls.table, len(dColumn), dColumn) global.Wlog.Debug(vlog) alterSlice := []string{} @@ -122,7 +122,7 @@ func (stcls *schemaTable) TableColumnNameCheck(checkTableList []string, logThrea v2 := []string{} for k, v22 := range v1 { v1k = k - if stcls.lowerCaseTableNames == "no" { + if stcls.caseSensitiveObjectName == "no" { v1k = strings.ToUpper(k) } v2 = v22 @@ -136,7 +136,7 @@ func (stcls *schemaTable) TableColumnNameCheck(checkTableList []string, logThrea v2 := []string{} for k, v22 := range v1 { v1k = k - if stcls.lowerCaseTableNames == "no" { + if stcls.caseSensitiveObjectName == "no" { v1k = strings.ToUpper(k) } v2 = v22 @@ -150,15 +150,15 @@ func (stcls *schemaTable) TableColumnNameCheck(checkTableList []string, logThrea if len(addColumn) == 0 && len(delColumn) == 0 { newCheckTableList = append(newCheckTableList, fmt.Sprintf("%s.%s", stcls.schema, stcls.table)) } else { - vlog = fmt.Sprintf("(%d) %s The %s table structure of the current source and destination is inconsistent, please check whether the current table structure is consistent. add:{%v} del:{%v}", logThreadSeq, event, fmt.Sprintf("%s.%s", stcls.schema, stcls.table), addColumn, delColumn) + vlog = fmt.Sprintf("(%d) %s The [%s] table structure of srcDB and dstDB are different, the extra columns: {%v}, the missing columns: {%v}", logThreadSeq, event, fmt.Sprintf("%s.%s", stcls.schema, stcls.table), addColumn, delColumn) global.Wlog.Error(vlog) abnormalTableList = append(abnormalTableList, fmt.Sprintf("%s.%s", stcls.schema, stcls.table)) } - continue - } - if len(addColumn) == 0 && len(delColumn) == 0 { + // yejr存疑：不要加continue，否则可能导致当检查到有个表中列定义不一致时，这里会被跳过忽略检查 + //continue } - vlog = fmt.Sprintf("(%d) %s The column that needs to be deleted in the target %s table %s.%s is {%v}", logThreadSeq, event, stcls.destDrive, stcls.schema, stcls.table, delColumn) + + vlog = fmt.Sprintf("(%d) %s Some columns that should be deleted from dstDSN {%s}, table {%s.%s}, columns {%v}", logThreadSeq, event, stcls.destDrive, stcls.schema, stcls.table, delColumn) global.Wlog.Debug(vlog) //先删除缺失的 if len(delColumn) > 0 { @@ -168,7 +168,7 @@ func (stcls *schemaTable) TableColumnNameCheck(checkTableList []string, logThrea delete(destColumnMap, v1) } } - vlog = fmt.Sprintf("(%d) %s The statement to delete a column in %s table %s.%s on the target side is {%v}", logThreadSeq, event, stcls.destDrive, stcls.schema, stcls.table, alterSlice) + vlog = fmt.Sprintf("(%d) %s The DROP SQL on Table {%s.%s} on dstDSN {%s} should be \"%v\"", logThreadSeq, event, stcls.schema, stcls.table, stcls.destDrive, alterSlice) global.Wlog.Debug(vlog) for k1, v1 := range sourceColumnSlice { lastcolumn := "" @@ -184,8 +184,8 @@ func (stcls *schemaTable) TableColumnNameCheck(checkTableList []string, logThrea case "dst": alterColumnData = destColumnMap[v1] default: - err = errors.New(fmt.Sprintf("unknown parameters")) - vlog = fmt.Sprintf("(%d) %s The validation mode of the correct table structure is not selected. error message is {%v}", logThreadSeq, event, err) + err = errors.New(fmt.Sprintf("unknown options")) + vlog = fmt.Sprintf("(%d) %s The option \"checkObject\" is set incorrectly, error: {%v}", logThreadSeq, event, err) global.Wlog.Error(vlog) return nil, nil, err } @@ -202,14 +202,14 @@ func (stcls *schemaTable) TableColumnNameCheck(checkTableList []string, logThrea case "no": tableAbnormalBool = false default: - err = errors.New(fmt.Sprintf("unknown parameters")) - vlog = fmt.Sprintf("(%d) %s The validation mode of the correct table structure is not selected. error message is {%v}", logThreadSeq, event, err) + err = errors.New(fmt.Sprintf("unknown options")) + vlog = fmt.Sprintf("(%d) %s The option \"checkObject\" is set incorrectly, error: {%v}", logThreadSeq, event, err) global.Wlog.Error(vlog) return nil, nil, err } if tableAbnormalBool { modifySql := dbf.DataAbnormalFix().FixAlterColumnSqlDispos("modify", alterColumnData, k1, lastcolumn, v1, logThreadSeq) - vlog = fmt.Sprintf("(%d) %s The column name of column %s of the source and target table %s.%s is the same, but the definition of the column is inconsistent, and a modify statement is generated, and the modification statement is {%v}", logThreadSeq, v1, stcls.schema, stcls.table, modifySql) + vlog = fmt.Sprintf("(%d) %s The column definition of table {%s.%s} is different, and ALTER SQL is \"%s\"", logThreadSeq, v1, stcls.schema, stcls.table, modifySql) global.Wlog.Warn(vlog) alterSlice = append(alterSlice, modifySql) } @@ -260,7 +260,7 @@ func (stcls *schemaTable) TableColumnNameCheck(checkTableList []string, logThrea return nil, nil, err } addSql := dbf.DataAbnormalFix().FixAlterColumnSqlDispos("add", sourceColumnMap[v1], k1, lastcolumn, v1, logThreadSeq) - vlog = fmt.Sprintf("(%d) %s The column %s is missing in the %s table %s.%s on the target side, and the add statement is generated, and the add statement is {%v}", logThreadSeq, event, v1, stcls.destDrive, stcls.schema, stcls.table, addSql) + vlog = fmt.Sprintf("(%d) %s The column %s is missing in the dstDSN {%s} table %s.%s on the target side, and the add statement is generated, and the add statement is {%v}", logThreadSeq, event, v1, stcls.destDrive, stcls.schema, stcls.table, addSql) global.Wlog.Warn(vlog) alterSlice = append(alterSlice, addSql) delete(destColumnMap, v1) @@ -275,18 +275,21 @@ func (stcls *schemaTable) TableColumnNameCheck(checkTableList []string, logThrea vlog = fmt.Sprintf("(%d) %s The table structure consistency check of table %s is completed.", logThreadSeq, event, v) global.Wlog.Debug(vlog) if len(sqlS) > 0 { - vlog = fmt.Sprintf("(%d) %s Start to repair the statement in %s table %s on the target side according to the specified repair method. The repair statement is {%v}.", logThreadSeq, event, stcls.destDrive, v, sqlS) + vlog = fmt.Sprintf("(%d) %s Start to repair the statement in dstDSN {%s} table %s on the target side according to the specified repair method. The repair statement is {%v}.", logThreadSeq, event, stcls.destDrive, v, sqlS) global.Wlog.Debug(vlog) if err = ApplyDataFix(sqlS, stcls.datefix, stcls.sfile, stcls.destDrive, stcls.djdbc, logThreadSeq); err != nil { return nil, nil, err } - vlog = fmt.Sprintf("(%d) %s Target side %s table %s repair statement application is completed.", logThreadSeq, event, stcls.destDrive, v) + vlog = fmt.Sprintf("(%d) %s dstDSN {%s} table %s repair statement application is completed.", logThreadSeq, event, stcls.destDrive, v) global.Wlog.Debug(vlog) } } - vlog = fmt.Sprintf("(%d) %s The consistency information check of the source and target table structure and column information is completed", logThreadSeq, event) + vlog = fmt.Sprintf("(%d) %s The table structure checksum of srcDSN and dstDSN completed", logThreadSeq, event) global.Wlog.Info(vlog) + // 这里返回结果时，即便是检查模式为data，但当发现表结构不一致时 + // 会被加入abnormalTableList，而不会加入newCheckTableList + // 也就是当表结构不一致时，该表会被忽略执行数据校验 return newCheckTableList, abnormalTableList, nil } @@ -316,13 +319,13 @@ func (stcls *schemaTable) tableIndexAlgorithm(indexType map[string][]string) (st } //有单列索引存在 - if len(indexType["mui_single"]) >= 1 { - return "mui_single", indexType["mui_single"] + if len(indexType["mul_single"]) >= 1 { + return "mul_single", indexType["mul_single"] } //有无单列普通索引，和多列普通索引，选择多列普通索引 - if len(indexType["mui_multiseriate"]) > 1 { - return "mui_multiseriate", indexType["mui_multiseriate"] + if len(indexType["mul_multiseriate"]) > 1 { + return "mul_multiseriate", indexType["mul_multiseriate"] } } else { var err = errors.New("Missing indexes") @@ -451,28 +454,29 @@ func (stcls *schemaTable) SchemaTableFilter(logThreadSeq1, logThreadSeq2 int64) dbCheckNameList map[string]int err error ) - fmt.Println("-- gt-checksum init check table name -- ") - vlog = fmt.Sprintf("(%d) Start to init schema.table info.", logThreadSeq1) + fmt.Println("gt-checksum is opening check tables") + vlog = fmt.Sprintf("(%d) Obtain schema.table info", logThreadSeq1) global.Wlog.Info(vlog) //获取当前数据库信息列表 - tc := dbExec.TableColumnNameStruct{Table: stcls.table, Drive: stcls.sourceDrive, Db: stcls.sourceDB, IgnoreTable: stcls.ignoreTable, LowerCaseTableNames: stcls.lowerCaseTableNames} - vlog = fmt.Sprintf("(%d) query check database list info.", logThreadSeq1) + tc := dbExec.TableColumnNameStruct{Table: stcls.table, Drive: stcls.sourceDrive, Db: stcls.sourceDB, IgnoreTable: stcls.ignoreTable, CaseSensitiveObjectName: stcls.caseSensitiveObjectName} + vlog = fmt.Sprintf("(%d) Obtain databases list", logThreadSeq1) global.Wlog.Debug(vlog) if dbCheckNameList, err = tc.Query().DatabaseNameList(stcls.sourceDB, logThreadSeq2); err != nil { return f, err } - vlog = fmt.Sprintf("(%d) checksum database list message is {%s}", logThreadSeq1, dbCheckNameList) + vlog = fmt.Sprintf("(%d) Databases list: {%s}", logThreadSeq1, dbCheckNameList) global.Wlog.Debug(vlog) //判断校验的库是否为空，为空则退出 if len(dbCheckNameList) == 0 { - vlog = fmt.Sprintf("(%d) source %s query Schema list is empty", logThreadSeq1, stcls.sourceDrive) + vlog = fmt.Sprintf("(%d) Databases of srcDSN {%s} is empty, please check if the \"tables\" option is correct", logThreadSeq1, stcls.sourceDrive) global.Wlog.Error(vlog) return f, nil } schema := stcls.FuzzyMatchingDispos(dbCheckNameList, stcls.table, logThreadSeq1) if len(schema) == 0 { - vlog = fmt.Sprintf("(%d) source %s check Schema list is empty,Please check whether the database parameter is enabled for the table case setting.", logThreadSeq1, stcls.sourceDrive) - global.Wlog.Error(vlog) + vlog = fmt.Sprintf("(%d) Databases of srcDSN {%s} is empty, please check if the \"tables\" option is correct", logThreadSeq1, stcls.sourceDrive) + global.Wlog.Warn(vlog) + // 当指定DB下的表为空时，只报告Warn，而非Error，因为可能该DB下没有表，或者表名写错了 return f, nil } ignoreSchema := stcls.FuzzyMatchingDispos(dbCheckNameList, stcls.ignoreTable, logThreadSeq1) @@ -484,7 +488,7 @@ func (stcls *schemaTable) SchemaTableFilter(logThreadSeq1, logThreadSeq2 int64) for k, _ := range schema { f = append(f, k) } - vlog = fmt.Sprintf("(%d) schema.table {%s} init sccessfully, num [%d].", logThreadSeq1, f, len(f)) + vlog = fmt.Sprintf("(%d) Obtain schema.table %s success, num [%d].", logThreadSeq1, f, len(f)) global.Wlog.Info(vlog) return f, nil } @@ -516,24 +520,24 @@ func (stcls *schemaTable) SchemaTableAllCol(tableList []string, logThreadSeq, lo if strings.Contains(i, ".") { schema := strings.Split(i, ".")[0] table := strings.Split(i, ".")[1] - vlog = fmt.Sprintf("(%d) Start to query all column information of source DB %s table %s.%s", logThreadSeq, stcls.sourceDrive, schema, table) + vlog = fmt.Sprintf("(%d) Start to query all column information of srcDSN {%s} table %s.%s", logThreadSeq, stcls.sourceDrive, schema, table) global.Wlog.Debug(vlog) tc := dbExec.TableColumnNameStruct{Schema: schema, Table: table, Drive: stcls.sourceDrive} a, err = tc.Query().TableAllColumn(stcls.sourceDB, logThreadSeq2) if err != nil { return nil } - vlog = fmt.Sprintf("(%d) All column information query of source DB %s table %s.%s is completed", logThreadSeq, stcls.sourceDrive, schema, table) + vlog = fmt.Sprintf("(%d) All column information query of srcDSN {%s} table %s.%s is completed", logThreadSeq, stcls.sourceDrive, schema, table) global.Wlog.Debug(vlog) - vlog = fmt.Sprintf("(%d) Start to query all column information of dest DB %s table %s.%s", logThreadSeq, stcls.destDrive, schema, table) + vlog = fmt.Sprintf("(%d) Start to query all column information of dstDSN {%s} table %s.%s", logThreadSeq, stcls.destDrive, schema, table) global.Wlog.Debug(vlog) tc.Drive = stcls.destDrive b, err = tc.Query().TableAllColumn(stcls.destDB, logThreadSeq2) if err != nil { return nil } - vlog = fmt.Sprintf("(%d) All column information query of dest DB %s table %s.%s is completed", logThreadSeq, stcls.destDrive, schema, table) + vlog = fmt.Sprintf("(%d) All column information query of dstDSN {%s} table %s.%s is completed", logThreadSeq, stcls.destDrive, schema, table) global.Wlog.Debug(vlog) tableCol[fmt.Sprintf("%s_greatdbCheck_%s", schema, table)] = global.TableAllColumnInfoS{ SColumnInfo: interfToString(a), @@ -616,32 +620,33 @@ func (stcls *schemaTable) Trigger(dtabS []string, logThreadSeq, logThreadSeq2 in global.Wlog.Info(vlog) for _, i := range dtabS { i = strings.Split(i, ".")[0] - if stcls.lowerCaseTableNames == "yes" { + if stcls.caseSensitiveObjectName == "yes" { i = strings.ToUpper(i) z[i]++ } - if stcls.lowerCaseTableNames == "no" { + // yejr存疑，这段代码是否多余，或者和上一段代码是否冲突？ + if stcls.caseSensitiveObjectName == "no" { z[i]++ } } //校验触发器 for i, _ := range z { pods.Schema = stcls.schema - vlog = fmt.Sprintf("(%d) Start processing source DB %s data databases %s Trigger. to dispos it...", logThreadSeq, stcls.sourceDrive, stcls.schema) + vlog = fmt.Sprintf("(%d) Start processing dstDSN {%s} data databases %s Trigger. to dispos it...", logThreadSeq, stcls.sourceDrive, stcls.schema) global.Wlog.Debug(vlog) tc := dbExec.TableColumnNameStruct{Schema: i, Drive: stcls.sourceDrive} if sourceTrigger, err = tc.Query().Trigger(stcls.sourceDB, logThreadSeq2); err != nil { return } - vlog = fmt.Sprintf("(%d) Source DB %s data databases %s message is {%s}", logThreadSeq, stcls.sourceDrive, stcls.schema, sourceTrigger) + vlog = fmt.Sprintf("(%d) dstDSN {%s} data databases %s message is {%s}", logThreadSeq, stcls.sourceDrive, stcls.schema, sourceTrigger) global.Wlog.Debug(vlog) - vlog = fmt.Sprintf("(%d) Start processing dest DB %s data databases %s Trigger data. to dispos it...", logThreadSeq, stcls.destDrive, stcls.schema) + vlog = fmt.Sprintf("(%d) Start processing dstDSN {%s} databases %s Trigger data. to dispos it...", logThreadSeq, stcls.destDrive, stcls.schema) global.Wlog.Debug(vlog) tc.Drive = stcls.destDrive if destTrigger, err = tc.Query().Trigger(stcls.destDB, logThreadSeq2); err != nil { return } - vlog = fmt.Sprintf("(%d) Dest DB %s data databases %s message is {%s}", logThreadSeq, stcls.destDrive, stcls.schema, destTrigger) + vlog = fmt.Sprintf("(%d) dstDSN {%s} databases %s message is {%s}", logThreadSeq, stcls.destDrive, stcls.schema, destTrigger) global.Wlog.Debug(vlog) if len(sourceTrigger) == 0 && len(destTrigger) == 0 { vlog = fmt.Sprintf("(%d) The current original target data is empty, and the verification of this databases %s will be skipped", logThreadSeq, stcls.schema) @@ -662,10 +667,10 @@ func (stcls *schemaTable) Trigger(dtabS []string, logThreadSeq, logThreadSeq2 in for k, _ := range tmpM { pods.TriggerName = strings.ReplaceAll(strings.Split(k, ".")[1], "\"", "") if sourceTrigger[k] != destTrigger[k] { - pods.Differences = "yes" + pods.DIFFS = "yes" d = append(d, k) } else { - pods.Differences = "no" + pods.DIFFS = "no" c = append(c, k) } vlog = fmt.Sprintf("(%d) Complete the consistency check of the source target segment databases %s Trigger. normal databases message is {%s} num [%d] abnormal databases message is {%s} num [%d]", logThreadSeq, stcls.schema, c, len(c), d, len(d)) @@ -702,21 +707,21 @@ func (stcls *schemaTable) Proc(dtabS []string, logThreadSeq, logThreadSeq2 int64 } for schema, _ := range schemaMap { - vlog = fmt.Sprintf("(%d) Start processing source DB %s data databases %s Stored Procedure. to dispos it...", logThreadSeq, stcls.sourceDrive, stcls.schema) + vlog = fmt.Sprintf("(%d) Start processing srcDSN {%s} databases %s Stored Procedure. to dispos it...", logThreadSeq, stcls.sourceDrive, stcls.schema) global.Wlog.Debug(vlog) tc := dbExec.TableColumnNameStruct{Schema: schema, Drive: stcls.sourceDrive} if sourceProc, err = tc.Query().Proc(stcls.sourceDB, logThreadSeq2); err != nil { return } - vlog = fmt.Sprintf("(%d) Source DB %s data databases %s message is {%s}", logThreadSeq, stcls.sourceDrive, stcls.schema, sourceProc) + vlog = fmt.Sprintf("(%d) srcDSN {%s} databases %s message is {%s}", logThreadSeq, stcls.sourceDrive, stcls.schema, sourceProc) global.Wlog.Debug(vlog) tc.Drive = stcls.destDrive - vlog = fmt.Sprintf("(%d) Start processing dest DB %s data table %s Stored Procedure data. to dispos it...", logThreadSeq, stcls.destDrive, stcls.schema, stcls.table) + vlog = fmt.Sprintf("(%d) Start processing dstDSN {%s} table %s Stored Procedure data. to dispos it...", logThreadSeq, stcls.destDrive, stcls.schema, stcls.table) global.Wlog.Debug(vlog) if destProc, err = tc.Query().Proc(stcls.destDB, logThreadSeq2); err != nil { return } - vlog = fmt.Sprintf("(%d) Dest DB %s data databases %s message is {%s}", logThreadSeq, stcls.destDrive, stcls.schema, destProc) + vlog = fmt.Sprintf("(%d) dstDSN {%s} databases %s message is {%s}", logThreadSeq, stcls.destDrive, stcls.schema, destProc) global.Wlog.Debug(vlog) if len(sourceProc) == 0 && len(destProc) == 0 { vlog = fmt.Sprintf("(%d) The current original target data is empty, and the verification of this databases %s will be skipped", logThreadSeq, stcls.schema) @@ -746,21 +751,21 @@ func (stcls *schemaTable) Proc(dtabS []string, logThreadSeq, logThreadSeq2 int64 if stcls.sourceDrive != stcls.destDrive { if v == 2 { pods.ProcName = k - pods.Differences = "no" + pods.DIFFS = "no" c = append(c, k) } else { pods.ProcName = k - pods.Differences = "yes" + pods.DIFFS = "yes" d = append(d, k) } } else { if sourceProc[k] != destProc[k] { pods.ProcName = k - pods.Differences = "yes" + pods.DIFFS = "yes" d = append(d, k) } else { pods.ProcName = k - pods.Differences = "no" + pods.DIFFS = "no" c = append(c, k) } } @@ -799,22 +804,22 @@ func (stcls *schemaTable) Func(dtabS []string, logThreadSeq, logThreadSeq2 int64 } for schema, _ := range schemaMap { - vlog = fmt.Sprintf("(%d) Start processing source DB %s data databases %s Stored Function. to dispos it...", logThreadSeq, stcls.sourceDrive, stcls.schema) + vlog = fmt.Sprintf("(%d) Start processing srcDSN {%s} databases %s Stored Function. to dispos it...", logThreadSeq, stcls.sourceDrive, stcls.schema) global.Wlog.Debug(vlog) tc := dbExec.TableColumnNameStruct{Schema: schema, Drive: stcls.sourceDrive} if sourceFunc, err = tc.Query().Func(stcls.sourceDB, logThreadSeq2); err != nil { return } - vlog = fmt.Sprintf("(%d) Source DB %s data databases %s message is {%s}", logThreadSeq, stcls.sourceDrive, stcls.schema, sourceFunc) + vlog = fmt.Sprintf("(%d) srcDSN {%s} databases %s message is {%s}", logThreadSeq, stcls.sourceDrive, stcls.schema, sourceFunc) global.Wlog.Debug(vlog) tc.Drive = stcls.destDrive - vlog = fmt.Sprintf("(%d) Start processing dest DB %s data table %s Stored Function data. to dispos it...", logThreadSeq, stcls.destDrive, stcls.schema, stcls.table) + vlog = fmt.Sprintf("(%d) Start processing dstDSN {%s} table %s Stored Function data. to dispos it...", logThreadSeq, stcls.destDrive, stcls.schema, stcls.table) global.Wlog.Debug(vlog) if destFunc, err = tc.Query().Func(stcls.destDB, logThreadSeq2); err != nil { return } - vlog = fmt.Sprintf("(%d) Dest DB %s data databases %s message is {%s}", logThreadSeq, stcls.destDrive, stcls.schema, destFunc) + vlog = fmt.Sprintf("(%d) dstDSN {%s} databases %s message is {%s}", logThreadSeq, stcls.destDrive, stcls.schema, destFunc) global.Wlog.Debug(vlog) if len(sourceFunc) == 0 && len(destFunc) == 0 { @@ -840,22 +845,22 @@ func (stcls *schemaTable) Func(dtabS []string, logThreadSeq, logThreadSeq2 int64 if stcls.sourceDrive != stcls.destDrive { //异构,只校验函数名 if v == 2 { pods.FuncName = k - pods.Differences = "no" + pods.DIFFS = "no" c = append(c, k) } else { pods.FuncName = k - pods.Differences = "yes" + pods.DIFFS = "yes" d = append(d, k) } } else { //相同架构，校验函数结构体 sv, dv = sourceFunc[k], destFunc[k] if sv != dv { pods.FuncName = k - pods.Differences = "yes" + pods.DIFFS = "yes" d = append(d, k) } else { pods.FuncName = k - pods.Differences = "no" + pods.DIFFS = "no" c = append(c, k) } } @@ -870,46 +875,6 @@ func (stcls *schemaTable) Func(dtabS []string, logThreadSeq, logThreadSeq2 int64 global.Wlog.Info(vlog) } -/* - 校验函数 -*/ - -//func (stcls *schemaTable) IndexDisposF(queryData []map[string]interface{}) ([]string, map[string][]string, map[string][]string) { -// nultiseriateIndexColumnMap := make(map[string][]string) -// multiseriateIndexColumnMap := make(map[string][]string) -// var PriIndexCol, uniIndexCol, mulIndexCol []string -// var indexName string -// for _, v := range queryData { -// var currIndexName = strings.ToUpper(v["indexName"].(string)) -// //判断唯一索引（包含主键索引和普通索引） -// if v["nonUnique"].(string) == "0" || v["nonUnique"].(string) == "UNIQUE" { -// if currIndexName == "PRIMARY" || v["columnKey"].(string) == "1" { -// if currIndexName != indexName { -// indexName = currIndexName -// } -// PriIndexCol = append(PriIndexCol, fmt.Sprintf("%s", v["columnName"])) -// } else { -// if currIndexName != indexName { -// indexName = currIndexName -// nultiseriateIndexColumnMap[indexName] = append(uniIndexCol, fmt.Sprintf("%s /*actions Column Type*/ %s", v["columnName"], v["columnType"])) -// } else { -// nultiseriateIndexColumnMap[indexName] = append(nultiseriateIndexColumnMap[indexName], fmt.Sprintf("%s /*actions Column Type*/ %s", v["columnName"], v["columnType"])) -// } -// } -// } -// //处理普通索引 -// if v["nonUnique"].(string) == "1" || (v["nonUnique"].(string) == "NONUNIQUE" && v["columnKey"].(string) == "0") { -// if currIndexName != indexName { -// indexName = currIndexName -// multiseriateIndexColumnMap[indexName] = append(mulIndexCol, fmt.Sprintf("%s /*actions Column Type*/ %s", v["columnName"], v["columnType"])) -// } else { -// multiseriateIndexColumnMap[indexName] = append(multiseriateIndexColumnMap[indexName], fmt.Sprintf("%s /*actions Column Type*/ %s", v["columnName"], v["columnType"])) -// } -// } -// } -// return PriIndexCol, nultiseriateIndexColumnMap, multiseriateIndexColumnMap -//} - func (stcls *schemaTable) Foreign(dtabS []string, logThreadSeq, logThreadSeq2 int64) { var ( vlog string @@ -929,7 +894,7 @@ func (stcls *schemaTable) Foreign(dtabS []string, logThreadSeq, logThreadSeq2 in for _, i := range dtabS { stcls.schema = strings.Split(i, ".")[0] stcls.table = strings.Split(i, ".")[1] - vlog = fmt.Sprintf("(%d) Start processing source DB %s data table %s.%s Foreign. to dispos it...", logThreadSeq, stcls.sourceDrive, stcls.schema, stcls.table) + vlog = fmt.Sprintf("(%d) Start processing srcDSN {%s} table %s.%s Foreign. to dispos it...", logThreadSeq, stcls.sourceDrive, stcls.schema, stcls.table) global.Wlog.Debug(vlog) pods.Schema = stcls.schema pods.Table = stcls.table @@ -937,17 +902,17 @@ func (stcls *schemaTable) Foreign(dtabS []string, logThreadSeq, logThreadSeq2 in if sourceForeign, err = tc.Query().Foreign(stcls.sourceDB, logThreadSeq2); err != nil { return } - vlog = fmt.Sprintf("(%d) Source DB %s data table %s.%s message is {%s}", logThreadSeq, stcls.sourceDrive, stcls.schema, stcls.table, sourceForeign) + vlog = fmt.Sprintf("(%d) srcDSN {%s} table %s.%s message is {%s}", logThreadSeq, stcls.sourceDrive, stcls.schema, stcls.table, sourceForeign) global.Wlog.Debug(vlog) - vlog = fmt.Sprintf("(%d) Start processing dest DB %s data table %s.%s Foreign. to dispos it...", logThreadSeq, stcls.destDrive, stcls.schema, stcls.table) + vlog = fmt.Sprintf("(%d) Start processing dstDSN {%s} table %s.%s Foreign. to dispos it...", logThreadSeq, stcls.destDrive, stcls.schema, stcls.table) global.Wlog.Debug(vlog) tc.Drive = stcls.destDrive if destForeign, err = tc.Query().Foreign(stcls.destDB, logThreadSeq2); err != nil { return } - vlog = fmt.Sprintf("(%d) Dest DB %s data table %s.%s message is {%s}", logThreadSeq, stcls.destDrive, stcls.schema, stcls.table, destForeign) + vlog = fmt.Sprintf("(%d) dstDSN {%s} table %s.%s message is {%s}", logThreadSeq, stcls.destDrive, stcls.schema, stcls.table, destForeign) global.Wlog.Debug(vlog) if len(sourceForeign) == 0 && len(destForeign) == 0 { vlog = fmt.Sprintf("(%d) The current original target data is empty, and the verification of this table %s.%s will be skipped", logThreadSeq, stcls.schema, stcls.table) @@ -967,10 +932,10 @@ func (stcls *schemaTable) Foreign(dtabS []string, logThreadSeq, logThreadSeq2 in global.Wlog.Debug(vlog) for k, _ := range tmpM { if sourceForeign[k] != destForeign[k] { - pods.Differences = "yes" + pods.DIFFS = "yes" d = append(d, k) } else { - pods.Differences = "no" + pods.DIFFS = "no" c = append(c, k) } } @@ -1002,18 +967,18 @@ func (stcls *schemaTable) Partitions(dtabS []string, logThreadSeq, logThreadSeq2 for _, i := range dtabS { stcls.schema = strings.Split(i, ".")[0] stcls.table = strings.Split(i, ".")[1] - vlog = fmt.Sprintf("(%d) Start processing source DB %s data table %s.%s partitions data. to dispos it...", logThreadSeq, stcls.sourceDrive, stcls.schema, stcls.table) + vlog = fmt.Sprintf("(%d) Start processing srcDSN {%s} table %s.%s partitions data. to dispos it...", logThreadSeq, stcls.sourceDrive, stcls.schema, stcls.table) global.Wlog.Debug(vlog) tc := dbExec.TableColumnNameStruct{Schema: stcls.schema, Table: stcls.table, Drive: stcls.sourceDrive} if sourcePartitions, err = tc.Query().Partitions(stcls.sourceDB, logThreadSeq2); err != nil { return } - vlog = fmt.Sprintf("(%d) Source DB %s data table %s.%s message is {%s}", logThreadSeq, stcls.sourceDrive, stcls.schema, stcls.table, sourcePartitions) + vlog = fmt.Sprintf("(%d) srcDSN {%s} table %s.%s message is {%s}", logThreadSeq, stcls.sourceDrive, stcls.schema, stcls.table, sourcePartitions) global.Wlog.Debug(vlog) tc.Drive = stcls.destDrive - vlog = fmt.Sprintf("(%d) Start processing dest DB %s data table %s.%s partitions data. to dispos it...", logThreadSeq, stcls.destDrive, stcls.schema, stcls.table) + vlog = fmt.Sprintf("(%d) Start processing dstDSN {%s} table %s.%s partitions data. to dispos it...", logThreadSeq, stcls.destDrive, stcls.schema, stcls.table) global.Wlog.Debug(vlog) if destPartitions, err = tc.Query().Partitions(stcls.destDB, logThreadSeq2); err != nil { return @@ -1042,11 +1007,11 @@ func (stcls *schemaTable) Partitions(dtabS []string, logThreadSeq, logThreadSeq2 global.Wlog.Debug(vlog) for k, _ := range tmpM { if strings.Join(strings.Fields(sourcePartitions[k]), "") != strings.Join(strings.Fields(destPartitions[k]), "") { - pods.Differences = "yes" + pods.DIFFS = "yes" d = append(d, k) } else { c = append(c, k) - pods.Differences = "no" + pods.DIFFS = "no" } } vlog = fmt.Sprintf("(%d) Complete the consistency check of the source target segment table %s.%s partitions. normal table message is {%s} num [%d] abnormal table message is {%s} num [%d]", logThreadSeq, stcls.schema, stcls.table, c, len(c), d, len(d)) @@ -1074,7 +1039,7 @@ func (stcls *schemaTable) Index(dtabS []string, logThreadSeq, logThreadSeq2 int6 seq, _ := strconv.Atoi(seqStr) return colName, seq } - + // 辅助函数：按序号排序列并返回纯列名 sortColumns = func(columns []string) []string { type ColumnInfo struct { @@ -1082,18 +1047,18 @@ func (stcls *schemaTable) Index(dtabS []string, logThreadSeq, logThreadSeq2 int6 seq int } var columnInfos []ColumnInfo - + // 提取列信息 for _, col := range columns { name, seq := extractColumnInfo(col) columnInfos = append(columnInfos, ColumnInfo{name: name, seq: seq}) } - + // 按序号排序 sort.Slice(columnInfos, func(i, j int) bool { return columnInfos[i].seq < columnInfos[j].seq }) - + // 返回排序后的纯列名 var result []string for _, col := range columnInfos { @@ -1101,7 +1066,7 @@ func (stcls *schemaTable) Index(dtabS []string, logThreadSeq, logThreadSeq2 int6 } return result } - + indexGenerate = func(smu, dmu map[string][]string, a *CheckSumTypeStruct, indexType string) []string { var cc, c, d []string dbf := dbExec.DataAbnormalFixStruct{ @@ -1112,7 +1077,7 @@ func (stcls *schemaTable) Index(dtabS []string, logThreadSeq, logThreadSeq2 int6 IndexType: indexType, DatafixType: stcls.datefix, } - + // 首先比较索引名称 for k := range smu { c = append(c, k) @@ -1120,7 +1085,7 @@ func (stcls *schemaTable) Index(dtabS []string, logThreadSeq, logThreadSeq2 int6 for k := range dmu { d = append(d, k) } - + // 如果索引名称不同，生成修复SQL if a.CheckMd5(strings.Join(c, ",")) != a.CheckMd5(strings.Join(d, ",")) { e, f := a.Arrcmp(c, d) @@ -1145,10 +1110,10 @@ func (stcls *schemaTable) Index(dtabS []string, logThreadSeq, logThreadSeq2 int6 } else { cc = append(cc, fmt.Sprintf("ALTER TABLE `%s`.`%s` DROP INDEX `%s`;", stcls.schema, stcls.table, k)) } - + // 2. 获取排序后的纯列名 sortedColumns := sortColumns(sColumns) - + // 3. 生成创建索引的SQL if indexType == "pri" { cc = append(cc, fmt.Sprintf("ALTER TABLE `%s`.`%s` ADD PRIMARY KEY(%s);", @@ -1167,8 +1132,8 @@ func (stcls *schemaTable) Index(dtabS []string, logThreadSeq, logThreadSeq2 int6 return cc } ) - - fmt.Println("-- gt-checksum checksum table index info -- ") + + fmt.Println("gt-checksum is opening indexes") event = fmt.Sprintf("[%s]", "check_table_index") //校验索引 vlog = fmt.Sprintf("(%d) %s start init check source and target DB index Column. to check it...", logThreadSeq, event) @@ -1177,52 +1142,52 @@ func (stcls *schemaTable) Index(dtabS []string, logThreadSeq, logThreadSeq2 int6 stcls.schema = strings.Split(i, ".")[0] stcls.table = strings.Split(i, ".")[1] idxc := dbExec.IndexColumnStruct{Schema: stcls.schema, Table: stcls.table, Drivce: stcls.sourceDrive} - vlog = fmt.Sprintf("(%d) %s Start processing source DB %s data table %s.%s index column data. to dispos it...", logThreadSeq, event, stcls.sourceDrive, stcls.schema, stcls.table) + vlog = fmt.Sprintf("(%d) %s Start processing srcDSN {%s} table %s.%s index column data. to dispos it...", logThreadSeq, event, stcls.sourceDrive, stcls.schema, stcls.table) global.Wlog.Debug(vlog) squeryData, err := idxc.TableIndexColumn().QueryTableIndexColumnInfo(stcls.sourceDB, logThreadSeq2) if err != nil { - vlog = fmt.Sprintf("(%d) %s Querying the index column data of source %s database table %s failed, and the error message is {%v}", logThreadSeq, event, stcls.sourceDrive, i, err) + vlog = fmt.Sprintf("(%d) %s Querying the index column data of srcDSN {%s} database table %s failed, and the error message is {%v}", logThreadSeq, event, stcls.sourceDrive, i, err) global.Wlog.Error(vlog) return err } spri, suni, smul := idxc.TableIndexColumn().IndexDisposF(squeryData, logThreadSeq2) - vlog = fmt.Sprintf("(%d) %s The index column data of the source %s database table %s.%s is {primary:%v,unique key:%v,index key:%v}", - logThreadSeq, - event, - stcls.sourceDrive, + vlog = fmt.Sprintf("(%d) %s The index column data of the source %s database table %s.%s is {primary:%v,unique key:%v,index key:%v}", + logThreadSeq, + event, + stcls.sourceDrive, stcls.schema, stcls.table, - spri, - suni, + spri, + suni, smul) global.Wlog.Debug(vlog) idxc.Drivce = stcls.destDrive - vlog = fmt.Sprintf("(%d) %s Start processing dest DB %s data table %s.%s index column data. to dispos it...", logThreadSeq, event, stcls.destDrive, stcls.schema, stcls.table) + vlog = fmt.Sprintf("(%d) %s Start processing dstDSN {%s} table %s.%s index column data. to dispos it...", logThreadSeq, event, stcls.destDrive, stcls.schema, stcls.table) global.Wlog.Debug(vlog) dqueryData, err := idxc.TableIndexColumn().QueryTableIndexColumnInfo(stcls.destDB, logThreadSeq2) if err != nil { - vlog = fmt.Sprintf("(%d) %s Querying the index column data of dest %s database table %s failed, and the error message is {%v}", logThreadSeq, event, stcls.destDrive, i, err) + vlog = fmt.Sprintf("(%d) %s Querying the index column data of dstDSN {%s} database table %s failed, and the error message is {%v}", logThreadSeq, event, stcls.destDrive, i, err) global.Wlog.Error(vlog) return err } dpri, duni, dmul := idxc.TableIndexColumn().IndexDisposF(dqueryData, logThreadSeq2) - vlog = fmt.Sprintf("(%d) %s The index column data of the dest %s database table %s.%s is {primary:%v,unique key:%v,index key:%v}", - logThreadSeq, - event, - stcls.destDrive, + vlog = fmt.Sprintf("(%d) %s The index column data of the dest %s database table %s.%s is {primary:%v,unique key:%v,index key:%v}", + logThreadSeq, + event, + stcls.destDrive, stcls.schema, stcls.table, - dpri, - duni, + dpri, + duni, dmul) global.Wlog.Debug(vlog) var pods = Pod{ Datafix: stcls.datefix, CheckObject: "Index", - - Differences: "no", + + DIFFS: "no", Schema: stcls.schema, Table: stcls.table, } @@ -1246,20 +1211,20 @@ func (stcls *schemaTable) Index(dtabS []string, logThreadSeq, logThreadSeq2 int6 global.Wlog.Debug(vlog) // 应用并清空 sqlS if len(sqlS) > 0 { - pods.Differences = "yes" - + pods.DIFFS = "yes" + err := ApplyDataFix(sqlS, stcls.datefix, stcls.sfile, stcls.destDrive, stcls.djdbc, logThreadSeq) if err != nil { return err } sqlS = []string{} // 清空 sqlS 以便下一个表使用 } - + measuredDataPods = append(measuredDataPods, pods) vlog = fmt.Sprintf("(%d) %s The source target segment table %s.%s index column data verification is completed", logThreadSeq, event, stcls.schema, stcls.table) global.Wlog.Info(vlog) } - fmt.Println("-- gt-checksum report: Table index verification completed -- ") + fmt.Println("gt-checksum report: indexes verification completed") return nil } @@ -1273,14 +1238,14 @@ func (stcls *schemaTable) Struct(dtabS []string, logThreadSeq, logThreadSeq2 int event string ) event = fmt.Sprintf("[check_table_columns]") - fmt.Println("-- gt-checksum checksum table strcut info -- ") - vlog = fmt.Sprintf("(%d) %s begin check source and target struct. check object is {%v} num[%d]", logThreadSeq, event, dtabS, len(dtabS)) + fmt.Println("gt-checksum is checking table structure") + vlog = fmt.Sprintf("(%d) %s checking table structure of %v(num[%d]) from srcDSN and dstDSN", logThreadSeq, event, dtabS, len(dtabS)) global.Wlog.Info(vlog) normal, abnormal, err := stcls.TableColumnNameCheck(dtabS, logThreadSeq, logThreadSeq2) if err != nil { return err } - vlog = fmt.Sprintf("(%d) %s Complete the data consistency check of the source target segment table structure column. normal table message is {%s} num [%d], abnormal table message is {%s} num [%d].", logThreadSeq, event, normal, len(normal), abnormal, len(abnormal)) + vlog = fmt.Sprintf("(%d) %s Table structure and column checksum of srcDB and dstDB completed. The consistent result is {%s}(num [%d]), and the inconsistent result is {%s}(num [%d])", logThreadSeq, event, normal, len(normal), abnormal, len(abnormal)) global.Wlog.Debug(vlog) //输出校验结果信息 var pods = Pod{ @@ -1291,17 +1256,17 @@ func (stcls *schemaTable) Struct(dtabS []string, logThreadSeq, logThreadSeq2 int aa := strings.Split(i, ".") pods.Schema = aa[0] pods.Table = aa[1] - pods.Differences = "no" + pods.DIFFS = "no" measuredDataPods = append(measuredDataPods, pods) } for _, i := range abnormal { aa := strings.Split(i, ".") pods.Schema = aa[0] pods.Table = aa[1] - pods.Differences = "yes" + pods.DIFFS = "yes" measuredDataPods = append(measuredDataPods, pods) } - fmt.Println("-- gt-checksum report Table structure verification completed -- ") + fmt.Println("gt-checksum report: Table structure checksum completed") vlog = fmt.Sprintf("(%d) %s check source and target DB table struct complete", logThreadSeq, event) global.Wlog.Info(vlog) return nil @@ -1339,7 +1304,7 @@ func SchemaTableInit(m *inputArg.ConfigParameter) *schemaTable { destDrive: m.SecondaryL.DsnsV.DestDrive, sourceDB: sdb, destDB: ddb, - lowerCaseTableNames: m.SecondaryL.SchemaV.LowerCaseTableNames, + caseSensitiveObjectName: m.SecondaryL.SchemaV.CaseSensitiveObjectName, datefix: m.SecondaryL.RepairV.Datafix, sfile: m.SecondaryL.RepairV.FixFileFINE, djdbc: m.SecondaryL.DsnsV.DestJdbc, diff --git a/actions/schema_table_access_permissions.go b/actions/schema_table_access_permissions.go index f73be62de3745a0bae8f8b012a8fe867189e7419..529266f1ace5e92134092847d22fd9328a03f4e9 100644 --- a/actions/schema_table_access_permissions.go +++ b/actions/schema_table_access_permissions.go @@ -15,31 +15,31 @@ func (stcls *schemaTable) GlobalAccessPriCheck(logThreadSeq, logThreadSeq2 int64 err error StableList, DtableList bool ) - vlog = fmt.Sprintf("(%d) Start to get the source and target Global Access Permissions information and check whether they are consistent", logThreadSeq) + vlog = fmt.Sprintf("(%d) Obtain the global privileges for both the srcDB and dstDB, and check that they are set correctly", logThreadSeq) global.Wlog.Info(vlog) tc := dbExec.TableColumnNameStruct{Schema: stcls.schema, Table: stcls.table, Drive: stcls.sourceDrive, Datafix: stcls.datefix} - vlog = fmt.Sprintf("(%d) Start to get the source Global Access Permissions information and check whether they are consistent", logThreadSeq) + vlog = fmt.Sprintf("(%d) Obtain the global privileges for srcDB, and check that they are set correctly", logThreadSeq) global.Wlog.Debug(vlog) if StableList, err = tc.Query().GlobalAccessPri(stcls.sourceDB, logThreadSeq2); err != nil { return false } - vlog = fmt.Sprintf("(%d) The Global Access Permission verification of the source DB is completed, and the status of the global access permission is {%v}.", logThreadSeq, StableList) + vlog = fmt.Sprintf("(%d) The global privileges for srcDB check completed: {%v}.", logThreadSeq, StableList) global.Wlog.Debug(vlog) tc.Drive = stcls.destDrive - vlog = fmt.Sprintf("(%d) Start to get the dest Global Access Permissions information and check whether they are consistent", logThreadSeq) + vlog = fmt.Sprintf("(%d) Obtain the global privileges for dstDB, and check that they are set correctly", logThreadSeq) global.Wlog.Debug(vlog) if DtableList, err = tc.Query().GlobalAccessPri(stcls.destDB, logThreadSeq2); err != nil { return false } - vlog = fmt.Sprintf("(%d) The Global Access Permission verification of the dest DB is completed, and the status of the global access permission is {%v}.", logThreadSeq, DtableList) + vlog = fmt.Sprintf("(%d) The global privileges for dstDB check completed: {%v}.", logThreadSeq, DtableList) global.Wlog.Debug(vlog) if StableList && DtableList { - vlog = fmt.Sprintf("(%d) The verification of the global access permission of the source and destination is completed", logThreadSeq) + vlog = fmt.Sprintf("(%d) The global privileges for both srcDB and dstDB are check completed", logThreadSeq) global.Wlog.Info(vlog) return true } - vlog = fmt.Sprintf("(%d) Some global access permissions are missing at the source and destination, and verification cannot continue.", logThreadSeq) + vlog = fmt.Sprintf("(%d) Insufficient global privileges for srcDB or dstDB, unable to continue", logThreadSeq) global.Wlog.Error(vlog) return false } @@ -50,36 +50,36 @@ func (stcls *schemaTable) TableAccessPriCheck(checkTableList []string, logThread StableList, DtableList map[string]int newCheckTableList, abnormalTableList []string ) - vlog = fmt.Sprintf("(%d) Start to get the source and target table access permissions information and check whether they are consistent", logThreadSeq) + vlog = fmt.Sprintf("(%d) Obtain the privileges for tables access for both the srcDB and dstDB, and check that they are set correctly", logThreadSeq) global.Wlog.Info(vlog) tc := dbExec.TableColumnNameStruct{Schema: stcls.schema, Table: stcls.table, Drive: stcls.sourceDrive} - vlog = fmt.Sprintf("(%d) Start to get the source table access permissions information and check whether they are consistent", logThreadSeq) + vlog = fmt.Sprintf("(%d) Obtain the privileges for tables access for srcDB, and check that they are set correctly", logThreadSeq) global.Wlog.Debug(vlog) if StableList, err = tc.Query().TableAccessPriCheck(stcls.sourceDB, checkTableList, stcls.datefix, logThreadSeq2); err != nil { return nil, nil, err } if len(StableList) == 0 { - vlog = fmt.Sprintf("(%d) Complete the verification table permission verification of the source DB, the current verification table with permission is {%v}.", logThreadSeq, StableList) + vlog = fmt.Sprintf("(%d) The privileges for tables access for srcDB check failed: {%v}.", logThreadSeq, StableList) global.Wlog.Error(vlog) } else { - vlog = fmt.Sprintf("(%d) Complete the verification table permission verification of the source DB, the current verification table with permission is {%v}.", logThreadSeq, StableList) + vlog = fmt.Sprintf("(%d) The privileges for tables access for srcDB check completed: {%v}.", logThreadSeq, StableList) global.Wlog.Debug(vlog) } tc.Drive = stcls.destDrive - vlog = fmt.Sprintf("(%d) Start to get the dest table access permissions information and check whether they are consistent", logThreadSeq) + vlog = fmt.Sprintf("(%d) Obtain the privileges for tables access for dstDB, and check that they are set correctly", logThreadSeq) global.Wlog.Debug(vlog) if DtableList, err = tc.Query().TableAccessPriCheck(stcls.destDB, checkTableList, stcls.datefix, logThreadSeq2); err != nil { return nil, nil, err } if len(DtableList) == 0 { - vlog = fmt.Sprintf("(%d) Complete the verification table permission verification of the source DB, the current verification table with permission is {%v}.", logThreadSeq, DtableList) + vlog = fmt.Sprintf("(%d) The privileges for tables access for dstDB check failed: {%v}.", logThreadSeq, DtableList) global.Wlog.Error(vlog) } else { - vlog = fmt.Sprintf("(%d) Complete the verification table permission verification of the source DB, the current verification table with permission is {%v}.", logThreadSeq, DtableList) + vlog = fmt.Sprintf("(%d) The privileges for tables access for dstDB check completed: {%v}.", logThreadSeq, DtableList) global.Wlog.Debug(vlog) } - vlog = fmt.Sprintf("(%d) Start processing the difference of the table to be checked at the source and target.", logThreadSeq) + vlog = fmt.Sprintf("(%d) Start checking the differences between the tables in srcDB and dstDB", logThreadSeq) global.Wlog.Debug(vlog) for k, _ := range StableList { if _, ok := DtableList[k]; ok { @@ -88,7 +88,7 @@ func (stcls *schemaTable) TableAccessPriCheck(checkTableList []string, logThread abnormalTableList = append(abnormalTableList, k) } } - vlog = fmt.Sprintf("(%d) The difference processing of the table to be checked at the source and target ends is completed. normal table message is {%s} num [%d] abnormal table message is {%s} num [%d]", logThreadSeq, newCheckTableList, len(newCheckTableList), abnormalTableList, len(abnormalTableList)) + vlog = fmt.Sprintf("(%d) The checksum of srcDB and dstDB tables is complete. The [%d] consistent tables are: {%s}, and the [%d] inconsistent tables are: {%s}", logThreadSeq, len(newCheckTableList), newCheckTableList, len(abnormalTableList), abnormalTableList) global.Wlog.Info(vlog) return newCheckTableList, abnormalTableList, nil } diff --git a/actions/table_count_check.go b/actions/table_count_check.go index d6e940460ab2fcb69e515d4e6e8c497f10042a20..4fae1d565cd9cc667ce3c3f9a80adb81aa290ae5 100644 --- a/actions/table_count_check.go +++ b/actions/table_count_check.go @@ -65,19 +65,19 @@ func (sp *SchedulePlan) DoCountDataCheck() { Schema: schema, Table: table, CheckObject: sp.checkObject, - CheckMod: sp.checkMod, + CheckMode: sp.checkMod, } vlog = fmt.Sprintf("(%d) Start to verify the total number of rows of table %s.%s source and target ...", logThreadSeq, schema, table) global.Wlog.Debug(vlog) if stmpTableCount == dtmpTableCount { vlog = fmt.Sprintf("(%d) Verify that the total number of rows at the source and destination of table %s.%s is consistent", logThreadSeq, schema, table) global.Wlog.Debug(vlog) - pods.Differences = "no" + pods.DIFFS = "no" pods.Rows = fmt.Sprintf("%d,%d", stmpTableCount, dtmpTableCount) } else { vlog = fmt.Sprintf("(%d) Verify that the total number of rows at the source and destination of table %s.%s is inconsistent.", logThreadSeq, schema, table) global.Wlog.Debug(vlog) - pods.Differences = "yes" + pods.DIFFS = "yes" pods.Rows = fmt.Sprintf("%d,%d", stmpTableCount, dtmpTableCount) } measuredDataPods = append(measuredDataPods, pods) diff --git a/actions/table_index_dispos.go b/actions/table_index_dispos.go index c3f557427ecee27f58f25aaf6272ce078e691e93..b6bb5fa1867b849fedab8b8992f8d5b1d1dee36f 100644 --- a/actions/table_index_dispos.go +++ b/actions/table_index_dispos.go @@ -311,17 +311,17 @@ func (sp *SchedulePlan) queryTableData(selectSql chanMap, diffQueryData chanDiff if sp.tableMaxRows%uint64(sp.chanrowCount) > 0 { barTotal += 1 } - sp.bar.NewOption(0, barTotal, "task") + sp.bar.NewOption(0, barTotal, "Processing") } } if sp.checkMod == "sample" { - sp.bar.NewOption(0, sp.sampDataGroupNumber, "task") + sp.bar.NewOption(0, sp.sampDataGroupNumber, "Processing") } for { select { case d, ok := <-sc: if ok { - sp.bar.NewOption(0, d, "task") + sp.bar.NewOption(0, d, "Processing") } case c, ok := <-selectSql: if !ok { @@ -443,15 +443,7 @@ func (sp *SchedulePlan) AbnormalDataDispos(diffQueryData chanDiffDataS, cc chanS idxc.Sqlwhere = c1.SqlWhere[sp.ddrive] idxc.TableColumn = colData.DColumnInfo dtt, _ := idxc.TableIndexColumn().GeneratingQueryCriteria(ddb, logThreadSeq) - //对不同数据库的的null处理 - //if aa.CheckMd5(stt) != aa.CheckMd5(dtt) { - // if strings.Contains(stt, "/*go actions columnData*//*") { - // stt = strings.ReplaceAll(stt, "/*go actions columnData*//*", "/*go actions columnData*//*") - // } - // if strings.Contains(dtt, "/*go actions columnData*//*") { - // dtt = strings.ReplaceAll(dtt, "/*go actions columnData*//*", "/*go actions columnData*//*") - // } - //} + if aa.CheckMd5(stt) != aa.CheckMd5(dtt) { add, del := aa.Arrcmp(strings.Split(stt, "/*go actions rowData*/"), strings.Split(dtt, "/*go actions rowData*/")) stt, dtt = "", "" @@ -464,7 +456,7 @@ func (sp *SchedulePlan) AbnormalDataDispos(diffQueryData chanDiffDataS, cc chanS } else if strings.HasPrefix(c1.indexColumnType, "uni") { dbf.IndexType = "uni" } else { - dbf.IndexType = "mui" + dbf.IndexType = "mul" } if len(del) > 0 { vlog = fmt.Sprintf("(%d) Start to generate the delete statement of check table %s.%s.", logThreadSeq, c1.Schema, c1.Table) @@ -534,7 +526,7 @@ func (sp SchedulePlan) DataFixDispos(fixSQL chanString, logThreadSeq int64) { } } else { increSeq++ - sp.pods.Differences = "yes" + sp.pods.DIFFS = "yes" sqlSlice = append(sqlSlice, v) if increSeq == sp.fixTrxNum { var sqlSlice1 []string @@ -584,9 +576,9 @@ func (sp SchedulePlan) doIndexDataCheck() { sp.pods = &Pod{ Schema: sp.schema, Table: sp.table, - IndexCol: strings.TrimLeft(strings.Join(sp.columnName, ","), ","), - CheckMod: sp.checkMod, - Differences: "no", + IndexColumn: strings.TrimLeft(strings.Join(sp.columnName, ","), ","), + CheckMode: sp.checkMod, + DIFFS: "no", Datafix: sp.datafixType, } idxc.Drivce = sp.sdrive @@ -608,7 +600,10 @@ func (sp SchedulePlan) doIndexDataCheck() { } else { sp.tableMaxRows = B } - sp.pods.Rows = fmt.Sprintf("%d,%d", A, B) + // 重新查询精确行数 + sourceExactCount := sp.getExactRowCount(sp.sdbPool, sp.schema, sp.table, logThreadSeq) + targetExactCount := sp.getExactRowCount(sp.ddbPool, sp.schema, sp.table, logThreadSeq) + sp.pods.Rows = fmt.Sprintf("%d,%d", sourceExactCount, targetExactCount) var scheduleCount = make(chan int64, 1) go sp.indexColumnDispos(sqlWhere, selectColumnStringM) go sp.queryTableSql(sqlWhere, selectSql, tableColumn, scheduleCount, logThreadSeq) diff --git a/actions/table_no_Index_dispos.go b/actions/table_no_Index_dispos.go index bf2996b86b70b060db4fb308c5fe90f8fee55222..ebe9d453a907e7f71dd5315c2f01dcb3b8d737e0 100644 --- a/actions/table_no_Index_dispos.go +++ b/actions/table_no_Index_dispos.go @@ -97,8 +97,8 @@ func (sp *SchedulePlan) DataFixSql(tmpAnDateMap <-chan map[string]string, pods * rowData = ki sqlType = vi //noIndexD <- struct{}{} - pods.Differences = "yes" - dbf.IndexType = "mui" + pods.DIFFS = "yes" + dbf.IndexType = "mul" //go func() { // defer func() { // <-noIndexD @@ -156,7 +156,7 @@ func (sp *SchedulePlan) FixSqlExec(sqlStrExec <-chan string, logThreadSeq int64) global.Wlog.Debug(vlog) colData := sp.tableAllCol[fmt.Sprintf("%s_greatdbCheck_%s", sp.schema, sp.table)] dbf := dbExec.DataAbnormalFixStruct{Schema: sp.schema, Table: sp.table, ColData: colData.DColumnInfo, SourceDevice: sp.ddrive} - dbf.IndexColumnType = "mui" + dbf.IndexColumnType = "mul" for { select { case v, ok := <-sqlStrExec: @@ -322,9 +322,9 @@ func (sp *SchedulePlan) SingleTableCheckProcessing(chanrowCount int, logThreadSe global.Wlog.Info(vlog) barTableRow := sp.NoIndexTableCount(logThreadSeq) pods := Pod{Schema: sp.schema, Table: sp.table, - IndexCol: "noIndex", - CheckMod: sp.checkMod, - Differences: "no", + IndexColumn: "noIndex", + CheckMode: sp.checkMod, + DIFFS: "no", Datafix: sp.datafixType, } sp.bar.NewOption(0, barTableRow, "rows") @@ -396,9 +396,28 @@ func (sp *SchedulePlan) SingleTableCheckProcessing(chanrowCount int, logThreadSe sp.bar.Finish() sp.FixSqlExec(sqlStrExec, int64(logThreadSeq)) //输出校验结果信息 - pods.Rows = fmt.Sprintf("%v", maxTableCount) + // 重新查询精确行数 + sourceExactCount := sp.getExactRowCount(sp.sdbPool, sp.schema, sp.table, logThreadSeq) + targetExactCount := sp.getExactRowCount(sp.ddbPool, sp.schema, sp.table, logThreadSeq) + pods.Rows = fmt.Sprintf("%d,%d", sourceExactCount, targetExactCount) measuredDataPods = append(measuredDataPods, pods) vlog = fmt.Sprintf("(%d) No index table %s.%s The data consistency check of the original target end is completed", logThreadSeq, sp.schema, sp.table) global.Wlog.Info(vlog) - fmt.Println(fmt.Sprintf("%s.%s 校验完成", sp.schema, sp.table)) + fmt.Println(fmt.Sprintf("table %s.%s checksum complete", sp.schema, sp.table)) +} +// getExactRowCount 查询表的精确行数 +func (sp *SchedulePlan) getExactRowCount(dbPool *global.Pool, schema, table string, logThreadSeq int64) int64 { + db := dbPool.Get(logThreadSeq) + defer dbPool.Put(db, logThreadSeq) + + var count int64 + query := fmt.Sprintf("SELECT COUNT(*) FROM %s.%s", schema, table) + err := db.QueryRow(query).Scan(&count) + if err != nil { + // 如果查询失败，返回0 + vlog := fmt.Sprintf("(%d) Failed to get exact row count for %s.%s: %v", logThreadSeq, schema, table, err) + global.Wlog.Error(vlog) + return 0 + } + return count } diff --git a/actions/table_query_concurrency.go b/actions/table_query_concurrency.go index 79eed81770f9e47a7c243ec1b8b7faa541ee9582..45a3487a0f0a867d710d5201e1d7dceb0f08fd2d 100644 --- a/actions/table_query_concurrency.go +++ b/actions/table_query_concurrency.go @@ -11,7 +11,7 @@ import ( ) type SchedulePlan struct { - singleIndexChanRowCount, jointIndexChanRowCount, mqQueueDepth int + chunkSize, mqQueueDepth int schema, table string //待校验库名、表名 columnName []string //待校验表的列名，有可能是多个 tmpTableDataFileDir string //临时表文件生成的相对路径 @@ -79,21 +79,15 @@ func (sp *SchedulePlan) Schedulingtasks() { } } if len(v) == 0 { //校验无索引表 - if sp.singleIndexChanRowCount <= sp.jointIndexChanRowCount { - sp.chanrowCount = sp.singleIndexChanRowCount - } else { - sp.chanrowCount = sp.jointIndexChanRowCount - } + sp.chanrowCount = sp.chunkSize logThreadSeq := rand.Int63() sp.SingleTableCheckProcessing(sp.chanrowCount, logThreadSeq) } else { //校验有索引的表 - if len(v) > 1 { //根据索引列数量觉得chanrowCount数 - sp.chanrowCount = sp.jointIndexChanRowCount - } else { - sp.chanrowCount = sp.singleIndexChanRowCount - } + sp.chanrowCount = sp.chunkSize sp.columnName = v - fmt.Println(fmt.Sprintf("begin checkSum index table %s.%s", sp.schema, sp.table)) + // 开始新表的进度显示 + tableName := fmt.Sprintf("begin checkSum index table %s.%s", sp.schema, sp.table) + sp.bar.NewTableProgress(tableName) sp.doIndexDataCheck() fmt.Println() fmt.Println(fmt.Sprintf("table %s.%s checksum complete", sp.schema, sp.table)) @@ -108,8 +102,7 @@ func CheckTableQuerySchedule(sdb, ddb *global.Pool, tableIndexColumnMap map[stri concurrency: m.SecondaryL.RulesV.ParallelThds, sdbPool: sdb, ddbPool: ddb, - singleIndexChanRowCount: m.SecondaryL.RulesV.ChanRowCount, - jointIndexChanRowCount: m.SecondaryL.RulesV.ChanRowCount, + chunkSize: m.SecondaryL.RulesV.ChanRowCount, tableIndexColumnMap: tableIndexColumnMap, tableAllCol: tableAllCol, datafixType: m.SecondaryL.RepairV.Datafix, diff --git a/actions/table_sample_check.go b/actions/table_sample_check.go index ba76795a3587f871b517bc81587976810d2805e3..453f7c47c2036e20805c9431a2ee8d3d322dc146 100644 --- a/actions/table_sample_check.go +++ b/actions/table_sample_check.go @@ -1,7 +1,6 @@ package actions import ( - "database/sql" "fmt" "gt-checksum/dbExec" "gt-checksum/global" @@ -10,632 +9,6 @@ import ( "time" ) -/* - 递归查询索引列数据，并按照单次校验块的大小来切割索引列数据，生成查询的where条件 -*/ -func (sp SchedulePlan) SampRecursiveIndexColumn(sqlWhere chanString, sdb, ddb *sql.DB, level, queryNum int, where string, selectColumn map[string]map[string]string, logThreadSeq int64) { - //var ( - // sqlwhere string //查询sql的where条件 - // indexColumnUniqueS []map[string]string //源目标端索引列数据的集合（有序的） - // indexColumnType string //索引列的数据类型 - // indexColumnIsNull bool //索引列数据类型是否允许为null - // z = make(map[string]int) //源目标端索引列数据的集合（无序的） - // zint []int //int类型的索引列集合，用于正序排序 - // zfloat []float64 //float类型的索引列集合，用于正序排序 - // zchar []string //char类型的索引列集合，用于正序排序 - // znull = make(map[string]int) //针对null的值的一个处理 - // office int //浮点类型的偏移量 - // d int //索引列每一行group重复值的累加值，临时变量 - // e, g string //定义每个chunk的初始值和结尾值,e为起始值，g为数据查询的动态指针值 - // vlog string //日志输出变量 - // sa, da []map[string]interface{} //原目标端索引列数据 - // err error //错误日志 - //) - ////获取索引列的数据类型 - //vlog = fmt.Sprintf("(%d) Start to get the index column data type and null value constraint of table [%v.%v] index column [%v]...", logThreadSeq, sp.schema, sp.table, sp.columnName[level]) - //global.Wlog.Info(vlog) - //a := sp.tableAllCol[fmt.Sprintf("%s_greatdbCheck_%s", sp.schema, sp.table)].SColumnInfo - //vlog = fmt.Sprintf("(%d) All column data information of table [%v.%v] is {%v}.", logThreadSeq, sp.schema, sp.table, a) - //global.Wlog.Debug(vlog) - //IntType := []string{"TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT"} - //floatType := []string{"FLOAT", "DOUBLE", "DECIMAL"} - // - //for _, i := range a { - // if i["columnName"] == sp.columnName[level] { - // ct := strings.Split(strings.ToUpper(i["dataType"]), "(")[0] - // if strings.Contains(strings.Join(IntType, ","), ct) { - // indexColumnType = "int" - // } else if strings.Contains(strings.Join(floatType, ","), ct) { - // office, _ = strconv.Atoi(strings.TrimSpace(strings.ReplaceAll(strings.Split(i["dataType"], ",")[1], ")", ""))) - // indexColumnType = "float" - // } else { - // indexColumnType = "char" - // } - // if i["isNull"] == "YES" { //判断当前索引列是否允许为null - // indexColumnIsNull = true - // } - // } - //} - //vlog = fmt.Sprintf("(%d) The data type of index column [%v] of table [%v.%v] is {%v} and the null constraint is {%v}", logThreadSeq, sp.columnName[level], sp.schema, sp.table, indexColumnType, indexColumnIsNull) - //global.Wlog.Debug(vlog) - //vlog = fmt.Sprintf("(%d) The index column data type and null value constraint acquisition of index column [%v] of table [%v.%v] is completed!!!", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - //global.Wlog.Info(vlog) - // - ////查询源目标端索引列数据 - //idxc := dbExec.IndexColumnStruct{Schema: sp.schema, Table: sp.table, ColumnName: sp.columnName, - // ChanrowCount: sp.chanrowCount, Drivce: sp.sdrive, SelectColumn: selectColumn[sp.sdrive]} - //vlog = fmt.Sprintf("(%d) Start to query the index column data of index column [%v] of source table [%v.%v]...", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - //global.Wlog.Info(vlog) - //for i := 1; i < 4; i++ { - // sa, err = idxc.TableIndexColumn().TmpTableColumnGroupDataDispos(sdb, where, sp.columnName[level], logThreadSeq) - // if err != nil { - // vlog = fmt.Sprintf("(%d) Failed to query the data of index column [%v] of source table [%v.%v] for the %v time.", logThreadSeq, sp.columnName[level], sp.schema, sp.table, i) - // global.Wlog.Error(vlog) - // if i == 3 { - // return - // } - // time.Sleep(5 * time.Second) - // } else { - // break - // } - //} - //vlog = fmt.Sprintf("(%d) The index column data of index column [%v] in source table [%v.%v] is {%v}", logThreadSeq, sp.columnName[level], sp.schema, sp.table, sa) - //global.Wlog.Debug(vlog) - //if len(sa) == 0 { - // vlog = fmt.Sprintf("(%d) The index column data of index column [%v] in source table [%v.%v] is empty.", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - // global.Wlog.Warn(vlog) - //} - //vlog = fmt.Sprintf("(%d) The index column data query of index column [%v] of source table [%v.%v] is completed!!!", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - //idxc.Drivce = sp.ddrive - //idxc.SelectColumn = selectColumn[sp.ddrive] - //vlog = fmt.Sprintf("(%d) Start to query the index column data of index column [%v] of dest table [%v.%v]...", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - //for i := 1; i < 4; i++ { - // da, err = idxc.TableIndexColumn().TmpTableColumnGroupDataDispos(ddb, where, sp.columnName[level], logThreadSeq) - // if err != nil { - // vlog = fmt.Sprintf("(%d) Failed to query the data of index column [%v] of dest table [%v.%v] for the %v time.", logThreadSeq, sp.columnName[level], sp.schema, sp.table, i) - // global.Wlog.Error(vlog) - // if i == 3 { - // return - // } - // time.Sleep(5 * time.Second) - // } else { - // break - // } - //} - //vlog = fmt.Sprintf("(%d) The index column data of index column [%v] in dest table [%v.%v] is {%v}", logThreadSeq, sp.columnName[level], sp.schema, sp.table, da) - //global.Wlog.Debug(vlog) - //if len(da) == 0 { - // vlog = fmt.Sprintf("(%d) The index column data of index column [%v] in dest table [%v.%v] is empty.", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - // global.Wlog.Warn(vlog) - //} - //vlog = fmt.Sprintf("(%d) The index column data query of index column [%v] of dest table [%v.%v] is completed!!!", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - // - ////原目标端索引列数据去重，并按照索引列数据类型进行分类合并索引列 - //vlog = fmt.Sprintf("(%d) Start merging the data of index column [%v] of source target segment table [%v.%v]...", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - //global.Wlog.Info(vlog) - //for _, i := range sa { - // c, _ := strconv.Atoi(fmt.Sprintf("%v", i["count"])) - // z[fmt.Sprintf("%v", i["columnName"])] = c - // switch indexColumnType { - // case "int": - // if indexColumnIsNull { - // if fmt.Sprintf("%v", i["columnName"]) != "" { - // zc, _ := strconv.Atoi(fmt.Sprintf("%v", i["columnName"])) - // zint = append(zint, zc) - // } else { - // znull[""] = c - // } - // } else { - // zc, _ := strconv.Atoi(fmt.Sprintf("%v", i["columnName"])) - // zint = append(zint, zc) - // } - // case "float": - // //处理null值 - // if indexColumnIsNull { - // if fmt.Sprintf("%v", i["columnName"]) != "" { - // zc, _ := strconv.ParseFloat(fmt.Sprintf("%v", i["columnName"]), office) - // zfloat = append(zfloat, zc) - // } else { - // znull[""] = c - // } - // } else { - // zc, _ := strconv.ParseFloat(fmt.Sprintf("%v", i["columnName"]), office) - // zfloat = append(zfloat, zc) - // } - // case "char": - // if indexColumnIsNull { - // if fmt.Sprintf("%v", i["columnName"]) != "" { - // zchar = append(zchar, fmt.Sprintf("%v", i["columnName"])) - // } else { - // znull[""] = c - // } - // } else { - // zchar = append(zchar, fmt.Sprintf("%v", i["columnName"])) - // } - // } - //} - //sa = nil - //for _, i := range da { - // c, _ := strconv.Atoi(fmt.Sprintf("%v", i["count"])) - // if _, ok := z[fmt.Sprintf("%v", i["columnName"])]; ok { - // if c > z[fmt.Sprintf("%v", i["columnName"])] { - // z[fmt.Sprintf("%v", i["columnName"])] = c - // } - // } else { - // z[fmt.Sprintf("%v", i["columnName"])] = c - // switch indexColumnType { - // case "int": - // if indexColumnIsNull { - // if fmt.Sprintf("%v", i["columnName"]) != "" { - // zc, _ := strconv.Atoi(fmt.Sprintf("%v", i["columnName"])) - // zint = append(zint, zc) - // } else { - // znull[""] = c - // } - // } else { - // zc, _ := strconv.Atoi(fmt.Sprintf("%v", i["columnName"])) - // zint = append(zint, zc) - // } - // case "float": - // if indexColumnIsNull { - // if fmt.Sprintf("%v", i["columnName"]) != "" { - // zc, _ := strconv.ParseFloat(fmt.Sprintf("%v", i["columnName"]), office) - // zfloat = append(zfloat, zc) - // } else { - // znull[""] = c - // } - // } else { - // zc, _ := strconv.ParseFloat(fmt.Sprintf("%v", i["columnName"]), office) - // zfloat = append(zfloat, zc) - // } - // case "char": - // if indexColumnIsNull { - // if fmt.Sprintf("%v", i["columnName"]) != "" { - // zchar = append(zchar, fmt.Sprintf("%v", i["columnName"])) - // } else { - // znull[""] = c - // } - // } else { - // zchar = append(zchar, fmt.Sprintf("%v", i["columnName"])) - // } - // } - // } - //} - //da = nil - //vlog = fmt.Sprintf("(%d) The data merge of the index column [%v] of the source target segment table [%v.%v] is completed!!!", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - //global.Wlog.Info(vlog) - // - //vlog = fmt.Sprintf("(%d) Start sorting the merged data of index column [%v] of source target segment table [%v.%v] in positive order...", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - //global.Wlog.Info(vlog) - ////针对索引类数据进行正序排序 - //switch indexColumnType { - //case "int": - // vlog = fmt.Sprintf("(%d) Start sorting index column data of type int,The index column data that needs to be sorted is [%v] ...", logThreadSeq, zint) - // global.Wlog.Debug(vlog) - // sort.Ints(zint) - // vlog = fmt.Sprintf("(%d) The index column data of int type is sorted, and the data after sorting in positive order is [%v] !!!", logThreadSeq, zint) - // global.Wlog.Debug(vlog) - // if _, ok := znull[""]; ok { - // vlog = fmt.Sprintf("(%d) The index column data of int type and index column data is null values.", logThreadSeq) - // global.Wlog.Debug(vlog) - // indexColumnUniqueS = append(indexColumnUniqueS, map[string]string{"columnName": fmt.Sprintf("%v", ""), "count": fmt.Sprintf("%v", z[fmt.Sprintf("%v", "")])}) - // } - // for _, i := range zint { - // indexColumnUniqueS = append(indexColumnUniqueS, map[string]string{"columnName": fmt.Sprintf("%v", i), "count": fmt.Sprintf("%v", z[fmt.Sprintf("%v", i)])}) - // } - // zint, z = nil, nil - //case "float": - // vlog = fmt.Sprintf("(%d) Start sorting index column data of type float,The index column data that needs to be sorted is [%v] ...", logThreadSeq, zfloat) - // global.Wlog.Debug(vlog) - // sort.Float64s(zfloat) - // vlog = fmt.Sprintf("(%d) The index column data of float type is sorted, and the data after sorting in positive order is [%v] !!!", logThreadSeq, zfloat) - // global.Wlog.Debug(vlog) - // if _, ok := znull[""]; ok { - // vlog = fmt.Sprintf("(%d) The index column data of float type and index column data is null values.", logThreadSeq) - // global.Wlog.Debug(vlog) - // indexColumnUniqueS = append(indexColumnUniqueS, map[string]string{"columnName": fmt.Sprintf("%v", ""), "count": fmt.Sprintf("%v", z[fmt.Sprintf("%v", "")])}) - // } - // for _, i := range zfloat { - // ii := strconv.FormatFloat(i, 'f', 2, 64) - // indexColumnUniqueS = append(indexColumnUniqueS, map[string]string{"columnName": ii, "count": fmt.Sprintf("%v", z[ii])}) - // } - // zfloat, z = nil, nil - //case "char": - // vlog = fmt.Sprintf("(%d) Start sorting index column data of type char,The index column data that needs to be sorted is [%v] ...", logThreadSeq, zchar) - // global.Wlog.Debug(vlog) - // sort.Strings(zchar) - // vlog = fmt.Sprintf("(%d) The index column data of char type is sorted, and the data after sorting in positive order is [%v] !!!", logThreadSeq, zchar) - // global.Wlog.Debug(vlog) - // if _, ok := znull[""]; ok { - // vlog = fmt.Sprintf("(%d) The index column data of char type and index column data is null values.", logThreadSeq) - // global.Wlog.Debug(vlog) - // indexColumnUniqueS = append(indexColumnUniqueS, map[string]string{"columnName": fmt.Sprintf("%v", ""), "count": fmt.Sprintf("%v", z[fmt.Sprintf("%v", "")])}) - // } - // for _, i := range zchar { - // indexColumnUniqueS = append(indexColumnUniqueS, map[string]string{"columnName": i, "count": fmt.Sprintf("%v", z[i])}) - // } - // zchar, z = nil, nil - //} - //vlog = fmt.Sprintf("(%d) The positive sequence sorting of the merged data of the index column [%v] of the source target segment table [%v.%v] is completed!!!", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - //global.Wlog.Info(vlog) - // - //vlog = fmt.Sprintf("(%d) Start to recursively process the where condition of index column [%v] of table [%v.%v] according to the size of a single check block...", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - //global.Wlog.Info(vlog) - ////处理原目标端索引列数据的集合，并按照单次校验数据块大小来进行数据截取，如果是多列索引，则需要递归查询截取 - //var startRowBool = false - //var firstRow int - ////次数 - // - //for f, b := range indexColumnUniqueS { - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v], the column value is [%v], and the number of repeated data in the column is [%v]", logThreadSeq, level, where, sp.columnName[level], f, b["columnName"], b["count"]) - // global.Wlog.Debug(vlog) - // //处理null值 - // if b["columnName"] == "" && f == 0 { - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v], Start processing null value data...", logThreadSeq, level, where, sp.columnName[level], f) - // global.Wlog.Debug(vlog) - // if where != "" { - // sqlwhere = fmt.Sprintf(" %s %s is null ", where, sp.columnName[level]) - // } else { - // sqlwhere = fmt.Sprintf(" %s is null ", sp.columnName[level]) - // } - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v], the query sql-where is [%v], Null value data processing is complete!!!", logThreadSeq, level, where, sp.columnName[level], f, sqlwhere) - // global.Wlog.Debug(vlog) - // sqlWhere <- sqlwhere - // sqlwhere = "" - // startRowBool = true - // continue - // } - // if f == 0 && b["columnName"] != "" { - // startRowBool = true - // } - // //获取联合索引或单列索引的首值 - // if startRowBool { - // e = fmt.Sprintf("%v", b["columnName"]) - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v], The starting value of the current index column is [%v].", logThreadSeq, level, where, sp.columnName[level], f, e) - // global.Wlog.Debug(vlog) - // firstRow = f - // startRowBool = false - // } - // - // //获取每行的count值,并将count值记录及每次动态的值 - // c, _ := strconv.Atoi(fmt.Sprintf("%v", b["count"])) - // g = fmt.Sprintf("%v", b["columnName"]) - // - // // group count(*)的值进行累加 - // d = d + c - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v], The accumulated value of the duplicate value of the current index column is [%v].", logThreadSeq, level, where, sp.columnName[level], f, d) - // global.Wlog.Debug(vlog) - // //判断行数累加值是否小于要校验的值，并且是最后一条索引列数据 - // if d < queryNum && f == len(indexColumnUniqueS)-1 { - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v], {end index column} {group index column cumulative value < single block checksum}.", logThreadSeq, level, where, sp.columnName[level], f) - // global.Wlog.Debug(vlog) - // if level == len(sp.columnName)-1 { //最后一列索引 - // g = fmt.Sprintf("%v", b["columnName"]) - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v],{end index column} {end row data} start dispos...", logThreadSeq, level, where, sp.columnName[level], f) - // global.Wlog.Debug(vlog) - // if where != "" { - // if e == g { //只有一行值且小于块校验值 - // sqlwhere = fmt.Sprintf(" %v and %v = %v ", where, sp.columnName[level], g) - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v],the query sql where is [%v],{Single row index column data} {group index column cumulative value < single block checksum}.", logThreadSeq, level, where, sp.columnName[level], f, sqlwhere) - // global.Wlog.Debug(vlog) - // } else { - // sqlwhere = fmt.Sprintf(" %v and %v > '%v' and %v <= '%v' ", where, sp.columnName[level], e, sp.columnName[level], g) - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v],the query sql where is [%v],{Multi-row index column data} {group index column cumulative value < single block checksum}.", logThreadSeq, level, where, sp.columnName[level], f, sqlwhere) - // global.Wlog.Debug(vlog) - // } - // } else { - // sqlwhere = fmt.Sprintf(" %v > '%v' and %v <= '%v' ", sp.columnName[level], e, sp.columnName[level], g) - // } - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v],the query sql where is [%v],{end index column} {end row data} dispos Finish!!!", logThreadSeq, level, where, sp.columnName[level], f, sqlwhere) - // global.Wlog.Debug(vlog) - // } else { //非最后索引列，但是是数据的最后一行，且小于校验的行数 - // if where != "" { - // sqlwhere = fmt.Sprintf(" %v and %v > '%v' and %v <= '%v' ", where, sp.columnName[level], e, sp.columnName[level], g) - // } else { - // sqlwhere = fmt.Sprintf(" %v >= '%v' and %v <= '%v' ", sp.columnName[level], e, sp.columnName[level], g) - // } - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v],the query sql where is [%v],{not end index column} {end row data}", logThreadSeq, level, where, sp.columnName[level], f, sqlwhere) - // global.Wlog.Debug(vlog) - // } - // sqlWhere <- sqlwhere - // } - // - // //判断行数累加值是否>=要校验的值 - // if d >= queryNum { - // //判断联合索引列深度 - // if level == len(sp.columnName)-1 { //单列索引或最后一列索引 - // g = fmt.Sprintf("%s", b["columnName"]) - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v], {end index column} {group index column cumulative value > single block checksum}.", logThreadSeq, level, where, sp.columnName[level], f) - // global.Wlog.Debug(vlog) - // if where != "" { //递归的第二层或其他层 - // if sqlwhere == "" { //首行数据 - // sqlwhere = fmt.Sprintf(" %v and %v >= '%v' and %v <= '%v' ", where, sp.columnName[level], e, sp.columnName[level], g) - // } else { //非首行数据 - // sqlwhere = fmt.Sprintf(" %v and %v > '%v' and %v <= '%v' ", where, sp.columnName[level], e, sp.columnName[level], g) - // } - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v], the query sqlwhere is [%v], {end index column} {group index column cumulative value > single block checksum}.", logThreadSeq, level, where, sp.columnName[level], f, sqlwhere) - // global.Wlog.Debug(vlog) - // } else { //where条件为空（第一次调用） - // if sqlwhere == "" { //首行数据 - // sqlwhere = fmt.Sprintf(" %v >= '%v' and %v <= '%v' ", sp.columnName[level], e, sp.columnName[level], g) - // } else { //非首行数据 - // sqlwhere = fmt.Sprintf(" %v > '%v' and %v <= '%v' ", sp.columnName[level], e, sp.columnName[level], g) - // } - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v], the query sqlwhere is [%v], {end index column} {group index column cumulative value > single block checksum}.", logThreadSeq, level, where, sp.columnName[level], f, sqlwhere) - // global.Wlog.Debug(vlog) - // } - // e = fmt.Sprintf("%s", b["columnName"]) - // sqlWhere <- sqlwhere - // } else { //非最后一列索引列 - // //判断当前索引列的重复值是否是校验数据块大小的两倍 - // if d/queryNum < 2 { //小于校验块的两倍，则直接输出当前索引列深度的条件 - // //第一层 - // if level == 0 { - // if f == firstRow { - // sqlwhere = fmt.Sprintf(" %v and %v >= '%v' and %v <= '%v' ", where, sp.columnName[level], e, sp.columnName[level], g) - // } else { - // sqlwhere = fmt.Sprintf(" %v and %v > '%v' and %v <= '%v' ", where, sp.columnName[level], e, sp.columnName[level], g) - // } - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v],the query sqlwhere is [%s], {not end index column} {group index column cumulative value / single block checksum < 2}.", logThreadSeq, level, where, sp.columnName[level], f, sqlwhere) - // global.Wlog.Debug(vlog) - // } - // if level > 0 && level < len(sp.columnName)-1 { - // if f == firstRow { - // sqlwhere = fmt.Sprintf(" %v and %v >= '%v' and %v <= '%v' ", where, sp.columnName[level], e, sp.columnName[level], g) - // } else { - // sqlwhere = fmt.Sprintf(" %v and %v > '%v' and %v <= '%v' ", where, sp.columnName[level], e, sp.columnName[level], g) - // } - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v],the query sqlwhere is [%s], {not end index column} {group index column cumulative value / single block checksum < 2}.", logThreadSeq, level, where, sp.columnName[level], f, sqlwhere) - // global.Wlog.Debug(vlog) - // } - // e = fmt.Sprintf("%s", b["columnName"]) - // sqlWhere <- sqlwhere - // } else { //大于校验块的两倍，递归进入下一层索引列进行处理 - // if where != "" { - // where = fmt.Sprintf(" %v and %v = '%v' ", where, sp.columnName[level], g) - // } else { - // where = fmt.Sprintf(" %v = '%v' ", sp.columnName[level], g) - // } - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v], {not end index column} {group index column cumulative value / single block checksum > 2}.", logThreadSeq, level, where, sp.columnName[level], f) - // global.Wlog.Debug(vlog) - // level++ //索引列层数递增 - // //进入下一层的索引计算 - // sp.recursiveIndexColumn(sqlWhere, sdb, ddb, level, queryNum, where, selectColumn, logThreadSeq) - // level-- //回到上一层 - // //递归处理结束后，处理where条件，将下一层的索引列条件去掉 - // if strings.Contains(strings.TrimSpace(where), sp.columnName[level]) { - // where = strings.TrimSpace(where[:strings.Index(where, sp.columnName[level])]) - // if strings.HasSuffix(where, "and") { - // where = strings.TrimSpace(where[:strings.LastIndex(where, "and")]) - // } - // } - // vlog = fmt.Sprintf("(%d) The current index column level is [%v],the where condition is [%v], the index column is [%v], the query sequence number is [%v], {not end index column} {group index column cumulative value / single block checksum > 2}.", logThreadSeq, level, where, sp.columnName[level], f) - // global.Wlog.Debug(vlog) - // e = fmt.Sprintf("%s", b["columnName"]) - // } - // } - // d = 0 //累加值清0 - // } - //} - //vlog = fmt.Sprintf("(%d) Recursively process the where condition of the index column [%v] of table [%v.%v] according to the size of the word check block!!!", logThreadSeq, sp.columnName[level], sp.schema, sp.table) - //global.Wlog.Info(vlog) -} - -/* - 计算源目标段表的最大行数 -*/ -//func (sp SchedulePlan) SampLimiterSeq(limitPag chan string, limitPagDone chan bool) { //定义变量 -// var ( -// schema = sp.schema -// table = sp.table -// columnName = sp.columnName -// chanrowCount = sp.chanrowCount -// maxTableCount uint64 -// schedulePlanCount uint64 -// stmpTableCount, dtmpTableCount uint64 -// err error -// vlog string -// ) -// time.Sleep(time.Nanosecond * 2) -// rand.Seed(time.Now().UnixNano()) -// logThreadSeq := rand.Int63() -// vlog = fmt.Sprintf("(%d) Check table %s.%s and start generating query sequence.", logThreadSeq, schema, table) -// global.Wlog.Info(vlog) -// -// vlog = fmt.Sprintf("(%d) The current verification table %s.%s single verification row number is [%d]", logThreadSeq, schema, table, chanrowCount) -// global.Wlog.Info(vlog) -// sdb := sp.sdbPool.Get(logThreadSeq) -// //查询原目标端的表总行数，并生成调度计划 -// idxc := dbExec.IndexColumnStruct{Schema: sp.schema, Table: sp.table, ColumnName: sp.columnName, ChanrowCount: sp.chanrowCount, Drivce: sp.sdrive} -// stmpTableCount, err = idxc.TableIndexColumn().TmpTableIndexColumnRowsCount(sdb, logThreadSeq) -// if err != nil { -// fmt.Println(err) -// } -// sp.sdbPool.Put(sdb, logThreadSeq) -// idxc.Drivce = sp.ddrive -// ddb := sp.ddbPool.Get(logThreadSeq) -// dtmpTableCount, err = idxc.TableIndexColumn().TmpTableIndexColumnRowsCount(ddb, logThreadSeq) -// if err != nil { -// fmt.Println(err) -// } -// sp.ddbPool.Put(ddb, logThreadSeq) -// if stmpTableCount > dtmpTableCount || stmpTableCount == dtmpTableCount { -// maxTableCount = stmpTableCount -// } else { -// maxTableCount = dtmpTableCount -// } -// //输出校验结果信息 -// pods := Pod{ -// Schema: schema, -// Table: table, -// IndexCol: strings.TrimLeft(strings.Join(columnName, ","), ","), -// CheckMod: sp.checkMod, -// Differences: "no", -// Datafix: sp.datafixType, -// } -// if stmpTableCount != dtmpTableCount { -// pods.Rows = fmt.Sprintf("%d|%d", stmpTableCount, dtmpTableCount) -// measuredDataPods = append(measuredDataPods, pods) -// } else { -// newMaxTableCount := maxTableCount //抽样比例后的总数值 -// if maxTableCount > uint64(chanrowCount) { -// newMaxTableCount = maxTableCount * uint64(sp.ratio) / 100 -// if chanrowCount > sp.concurrency { -// chanrowCount = chanrowCount / sp.concurrency -// } -// } -// if newMaxTableCount%uint64(chanrowCount) != 0 { -// schedulePlanCount = newMaxTableCount/uint64(chanrowCount) + 1 -// } else { -// schedulePlanCount = newMaxTableCount / uint64(chanrowCount) -// } -// tlog := fmt.Sprintf("(%d) There is currently index table %s.%s, the number of rows to be verified at a time is %d, and the number of rows to be verified is %d times", logThreadSeq, schema, table, chanrowCount, schedulePlanCount) -// global.Wlog.Info(tlog) -// var beginSeq int64 -// nanotime := int64(time.Now().Nanosecond()) -// rand.Seed(nanotime) -// for i := 0; i < int(schedulePlanCount); i++ { -// if newMaxTableCount > uint64(chanrowCount) { -// beginSeq = rand.Int63n(int64(maxTableCount)) -// } -// xlog := fmt.Sprintf("(%d) Verify table %s.%s The query sequence is written to the mq queue for the %d time, and the written information is {%s}", logThreadSeq, schema, table, i, fmt.Sprintf("%d,%d", beginSeq, maxTableCount)) -// global.Wlog.Info(xlog) -// limitPag <- fmt.Sprintf("%d,%d", beginSeq, newMaxTableCount) -// beginSeq = beginSeq + int64(chanrowCount) -// } -// pods.Rows = fmt.Sprintf("%d,%d", maxTableCount, newMaxTableCount) -// measuredDataPods = append(measuredDataPods, pods) -// } -// limitPagDone <- true -// ylog := fmt.Sprintf("(%d) Verify table %s.%s Close the mq queue that stores the query sequence.", logThreadSeq, schema, table) -// global.Wlog.Info(ylog) -// close(limitPag) -// zlog := fmt.Sprintf("(%d) Verify that table %s.%s query sequence is generated. !!!", logThreadSeq, schema, table) -// global.Wlog.Info(zlog) -//} - -//func (sp SchedulePlan) SampIndexColumnDispos(sqlWhere chanString, selectColumn map[string]map[string]string, sqlWhereDone chanBool) { -// var ( -// schema = sp.schema -// table = sp.table -// columnName = sp.columnName -// chanrowCount = sp.chanrowCount -// maxTableCount, schedulePlanCount int -// ) -// time.Sleep(time.Nanosecond * 2) -// rand.Seed(time.Now().UnixNano()) -// logThreadSeq := rand.Int63() -// alog := fmt.Sprintf("(%d) Check table %s.%s and start generating query sequence.", logThreadSeq, schema, table) -// global.Wlog.Info(alog) -// -// clog := fmt.Sprintf("(%d) The current verification table %s.%s single verification row number is [%d]", logThreadSeq, schema, table, chanrowCount) -// global.Wlog.Info(clog) -// -// sdb := sp.sdbPool.Get(logThreadSeq) -// ddb := sp.ddbPool.Get(logThreadSeq) -// sp.recursiveIndexColumn(sqlWhere, sdb, ddb, 0, sp.chanrowCount, "", selectColumn, logThreadSeq) -// sp.sdbPool.Put(sdb, logThreadSeq) -// sp.ddbPool.Put(ddb, logThreadSeq) -// -// //输出校验结果信息 -// pods := Pod{ -// Schema: schema, -// Table: table, -// IndexCol: strings.TrimLeft(strings.Join(columnName, ","), ","), -// CheckMod: sp.checkMod, -// Differences: "no", -// Datafix: sp.datafixType, -// } -// if maxTableCount%chanrowCount != 0 { -// schedulePlanCount = maxTableCount/chanrowCount + 1 -// } else { -// schedulePlanCount = maxTableCount / chanrowCount -// } -// tlog := fmt.Sprintf("(%d) There is currently index table %s.%s, the number of rows to be verified at a time is %d, and the number of rows to be verified is %d times", logThreadSeq, schema, table, chanrowCount, schedulePlanCount) -// global.Wlog.Info(tlog) -// pods.Rows = fmt.Sprintf("%d,%d", maxTableCount, maxTableCount) -// measuredDataPods = append(measuredDataPods, pods) -// ylog := fmt.Sprintf("(%d) Verify table %s.%s Close the mq queue that stores the query sequence.", logThreadSeq, schema, table) -// global.Wlog.Info(ylog) -// zlog := fmt.Sprintf("(%d) Verify that table %s.%s query sequence is generated. !!!", logThreadSeq, schema, table) -// global.Wlog.Info(zlog) -// sqlWhereDone <- true -// close(sqlWhere) -//} - -/* -针对表的所有列的数据类型进行处理，将列类型转换成字符串，例如时间类型 -*/ -func (sp SchedulePlan) sampQueryTableSql(sqlWhere chanString, selectSql chanMap, sampDataGroupNumber uint64, cc1 global.TableAllColumnInfoS, logThreadSeq int64) { - var ( - vlog string - curry = make(chanStruct, sp.concurrency) - count uint64 - autoSeq int64 - sampleList = make(map[int64]int) - err error - ) - vlog = fmt.Sprintf("(%d) Start processing the block data verification query sql of the verification table ...", logThreadSeq) - global.Wlog.Debug(vlog) - - for i := 0; i < int(sp.sampDataGroupNumber); i++ { - rand.Seed(time.Now().UnixNano()) - c := rand.Int63n(int64(sp.tableMaxRows / uint64(sp.chanrowCount))) - if _, ok := sampleList[c]; !ok { - sampleList[c]++ - } - time.Sleep(1 * time.Nanosecond) - } - for { - select { - case c, ok := <-sqlWhere: - if !ok { - if len(curry) == 0 { - close(selectSql) - return - } - } else { - count++ - if _, ok1 := sampleList[int64(count)]; !ok1 { - continue - } - autoSeq++ - curry <- struct{}{} - sdb := sp.sdbPool.Get(logThreadSeq) - ddb := sp.ddbPool.Get(logThreadSeq) - //查询该表的列名和列信息 - go func(c1 string, sd, dd *sql.DB, sdbPool, ddbPool *global.Pool) { - var selectSqlMap = make(map[string]string) - defer func() { - sdbPool.Put(sd, logThreadSeq) - ddbPool.Put(dd, logThreadSeq) - <-curry - }() - //查询该表的列名和列信息 - idxc := dbExec.IndexColumnStruct{Schema: sp.schema, Table: sp.table, TableColumn: cc1.SColumnInfo, Sqlwhere: c1, Drivce: sp.sdrive} - lock.Lock() - selectSqlMap[sp.sdrive], err = idxc.TableIndexColumn().GeneratingQuerySql(sd, logThreadSeq) - if err != nil { - return - } - lock.Unlock() - idxc.Drivce = sp.ddrive - idxc.TableColumn = cc1.DColumnInfo - lock.Lock() - selectSqlMap[sp.ddrive], err = idxc.TableIndexColumn().GeneratingQuerySql(dd, logThreadSeq) - if err != nil { - return - } - lock.Unlock() - vlog = fmt.Sprintf("(%d) The block data verification query sql processing of the verification table is completed. !!!", logThreadSeq) - global.Wlog.Debug(vlog) - selectSql <- selectSqlMap - }(c, sdb, ddb, sp.sdbPool, sp.ddbPool) - } - } - } -} - /* 单表的数据循环校验 */ @@ -676,9 +49,9 @@ func (sp *SchedulePlan) sampSingleTableCheckProcessing(chanrowCount int, sampDat //sp.bar.NewOption(0, barTableRow) fmt.Println(A, B) pods := Pod{Schema: sp.schema, Table: sp.table, - IndexCol: "noIndex", - CheckMod: sp.checkMod, - Differences: "no", + IndexColumn: "noIndex", + CheckMode: sp.checkMod, + DIFFS: "no", Datafix: sp.datafixType, } @@ -766,7 +139,7 @@ func (sp *SchedulePlan) sampSingleTableCheckProcessing(chanrowCount int, sampDat measuredDataPods = append(measuredDataPods, pods) vlog = fmt.Sprintf("(%d) No index table %s.%s The data consistency check of the original target end is completed", logThreadSeq, sp.schema, sp.table) global.Wlog.Info(vlog) - fmt.Println(fmt.Sprintf("%s.%s 校验完成", sp.schema, sp.table)) + fmt.Println(fmt.Sprintf("table %s.%s checksum complete", sp.schema, sp.table)) } /* @@ -795,19 +168,19 @@ func (sp *SchedulePlan) DoSampleDataCheck() { //输出校验结果信息 sp.pods = &Pod{ CheckObject: sp.checkObject, - CheckMod: sp.checkMod, - Differences: "no", + CheckMode: sp.checkMod, + DIFFS: "no", } if strings.Contains(k, "/*indexColumnType*/") { ki := strings.Split(k, "/*indexColumnType*/")[0] - sp.pods.IndexCol = strings.TrimLeft(strings.Join(v, ","), ",") + sp.pods.IndexColumn = strings.TrimLeft(strings.Join(v, ","), ",") sp.indexColumnType = strings.Split(k, "/*indexColumnType*/")[1] if strings.Contains(ki, "/*greatdbSchemaTable*/") { sp.schema = strings.Split(ki, "/*greatdbSchemaTable*/")[0] sp.table = strings.Split(ki, "/*greatdbSchemaTable*/")[1] } } else { - sp.pods.IndexCol = "no" + sp.pods.IndexColumn = "no" if strings.Contains(k, "/*greatdbSchemaTable*/") { sp.schema = strings.Split(k, "/*greatdbSchemaTable*/")[0] sp.table = strings.Split(k, "/*greatdbSchemaTable*/")[1] @@ -817,18 +190,8 @@ func (sp *SchedulePlan) DoSampleDataCheck() { sp.pods.Table = sp.table fmt.Println(fmt.Sprintf("begin checkSum table %s.%s", sp.schema, sp.table)) tableColumn := sp.tableAllCol[fmt.Sprintf("%s_greatdbCheck_%s", sp.schema, sp.table)] - //根据索引列数量觉得chanrowCount数 - if len(v) > 1 { - sp.chanrowCount = sp.jointIndexChanRowCount - } else if len(v) == 1 { - sp.chanrowCount = sp.singleIndexChanRowCount - } else { - if sp.singleIndexChanRowCount <= sp.jointIndexChanRowCount { - sp.chanrowCount = sp.singleIndexChanRowCount - } else { - sp.chanrowCount = sp.jointIndexChanRowCount - } - } + //根据索引列数量决定chanrowCount数 + sp.chanrowCount = sp.chunkSize sp.columnName = v //统计表的总行数 sdb := sp.sdbPool.Get(logThreadSeq) @@ -852,7 +215,7 @@ func (sp *SchedulePlan) DoSampleDataCheck() { if stmpTableCount != dtmpTableCount { vlog = fmt.Sprintf("(%d) Verify that the total number of rows at the source and destination of table %s.%s is inconsistent.", logThreadSeq, sp.schema, sp.table) global.Wlog.Debug(vlog) - sp.pods.Differences = "yes" + sp.pods.DIFFS = "yes" sp.pods.Rows = fmt.Sprintf("%d,%d", stmpTableCount, dtmpTableCount) measuredDataPods = append(measuredDataPods, *sp.pods) vlog = fmt.Sprintf("(%d) Check table %s.%s The total number of rows at the source and target end has been checked.", logThreadSeq, sp.schema, sp.table) @@ -878,7 +241,7 @@ func (sp *SchedulePlan) DoSampleDataCheck() { sp.pods.Sample = fmt.Sprintf("%d,%d", stmpTableCount, stmpTableCount/100*uint64(sp.ratio)) if len(v) == 0 { - sp.pods.IndexCol = "noIndex" + sp.pods.IndexColumn = "noIndex" sp.sampSingleTableCheckProcessing(sp.chanrowCount, sampDataGroupNumber, logThreadSeq) //measuredDataPods = append(measuredDataPods, pods) fmt.Println() @@ -888,7 +251,7 @@ func (sp *SchedulePlan) DoSampleDataCheck() { continue } //开始校验有索引表 - sp.pods.IndexCol = strings.TrimLeft(strings.Join(sp.columnName, ","), ",") + sp.pods.IndexColumn = strings.TrimLeft(strings.Join(sp.columnName, ","), ",") //获取索引列数据长度，处理索引列数据中有null或空字符串的问题 idxc = dbExec.IndexColumnStruct{Schema: sp.schema, Table: sp.table, ColumnName: sp.columnName, ChanrowCount: chanrowCount, Drivce: sp.sdrive, @@ -899,7 +262,7 @@ func (sp *SchedulePlan) DoSampleDataCheck() { var scheduleCount = make(chan int64, 1) go sp.recursiveIndexColumn(sqlWhere, sdb, ddb, 0, sp.chanrowCount, "", selectColumnStringM, logThreadSeq) - go sp.sampQueryTableSql(sqlWhere, selectSql, sampDataGroupNumber, tableColumn, logThreadSeq) + go sp.queryTableData(selectSql, diffQueryData, tableColumn, scheduleCount, logThreadSeq) go sp.AbnormalDataDispos(diffQueryData, fixSQL, logThreadSeq) sp.DataFixDispos(fixSQL, logThreadSeq) diff --git a/build-arm.sh b/build-arm.sh deleted file mode 100644 index 63f73fb937844963fdd10f0735201388117903f8..0000000000000000000000000000000000000000 --- a/build-arm.sh +++ /dev/null @@ -1,20 +0,0 @@ -set -x - -export PATH=$PATH:/usr/local/go/bin -export GO111MODULE=on -export GOPROXY=https://goproxy.cn -export CXXFLAGS="-stdlib=libstdc++" CC=/usr/bin/gcc CXX=/usr/bin/g++ - -vs=`cat ./inputArg/flagHelp.go| grep "app.Version"|awk -F "=" '{print $2}'|sed 's/\"//g'|sed 's/\/\/版本//g'|sed 's/ //g'` -OracleDrive="instantclient_11_2" -if [ ! -d "/usr/lcoal/$OracleDrive" ];then - cp -rpf Oracle/$OracleDrive /usr/lcoal/ -fi -export LD_LIBRARY_PATH=/usr/local/$OracleDrive:$LD_LIBRARY_PATH - -go build -o gt-checksum greatdbCheck.go -mkdir gt-checksum-${vs}-linux-aarch64 -cp -rpf Oracle/${OracleDrive} gt-checksum gc.conf gc.conf-simple relnotes docs README.md gt-checksum-${vs}-linux-aarch64 -tar zcf gt-checksum-${vs}-linux-aarch64.tar.gz gt-checksum-${vs}-linux-aarch64 -mkdir binary -mv gt-checksum-${vs}-linux-aarch64.tar.gz binary diff --git a/build-x86.sh b/build-x86.sh deleted file mode 100644 index 70ae6d9fb71949346653ec1579d60a3280eb8515..0000000000000000000000000000000000000000 --- a/build-x86.sh +++ /dev/null @@ -1,21 +0,0 @@ -set -x - -export PATH=$PATH:/usr/local/go/bin -export GO111MODULE=on -export GOPROXY=https://goproxy.cn -export CXXFLAGS="-stdlib=libstdc++" CC=/usr/bin/gcc CXX=/usr/bin/g++ - -vs=`cat ./inputArg/flagHelp.go| grep "app.Version"|awk -F "=" '{print $2}'|sed 's/\"//g'|sed 's/\/\/版本//g'|sed 's/ //g'` -OracleDrive="instantclient_11_2" -if [ ! -d "/usr/lcoal/${OracleDrive}" ];then - cp -rpf Oracle/${OracleDrive} /usr/lcoal/ -fi -export LD_LIBRARY_PATH=/usr/local/${OracleDrive}:$LD_LIBRARY_PATH - -go build -o gt-checksum greatdbCheck.go -chmod +x gt-checksum -mkdir gt-checksum-${vs}-linux-x86-64 -cp -rpf Oracle/${OracleDrive} gt-checksum gc.conf gc.conf-simple relnotes docs README.md gt-checksum-${vs}-linux-x86-64 -tar zcf gt-checksum-${vs}-linux-x86-64.tar.gz gt-checksum-${vs}-linux-x86-64 -mkdir binary -mv gt-checksum-${vs}-linux-x86-64.tar.gz binary \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..73a5ad842204e63da2785b99cbd7de44baefb16b --- /dev/null +++ b/build.sh @@ -0,0 +1,47 @@ +# +# build gt-checksum +# Requires go version 1.17 or higher +# +# run as: +# sh ./build.sh +# + +export PATH=$PATH:/usr/local/go/bin +export GO111MODULE=on +export GOPROXY=https://goproxy.cn +export CXXFLAGS="-stdlib=libstdc++" CC=/usr/bin/gcc CXX=/usr/bin/g++ + +vs=`cat ./inputArg/flagHelp.go| grep "app.Version"|awk -F "=" '{print $2}'|sed 's/\"//g'|sed 's/\/\/版本//g'|sed 's/ //g'` +OracleDrive="instantclient_11_2" + +# 自动适配CPU架构类型 +if [ ! -z "`which uname > /dev/null 2>&1`" ] ; then + arch=`uname -m` +elif [ ! -z "`echo $MACHTYPE`" ] ; then + arch=`echo $MACHTYPE|awk -F '-' '{print $1}'` +else + arch=x86_64 +fi + +rm -fr gt-checksum-${vs}-linux-${arch} release +mkdir -p gt-checksum-${vs}-linux-${arch} release + +echo -n "1. " +go version + +echo "2. Setting Oracle Library PATH" +if [ ! -d "/tmp/${OracleDrive}" ];then + tar xf Oracle/${OracleDrive}.tar.xz -C /tmp/ +fi +export LD_LIBRARY_PATH=/tmp/${OracleDrive}:$LD_LIBRARY_PATH + +echo "3. Compiling gt-checksum" +go build -o gt-checksum gt-checksum.go +chmod +x gt-checksum +echo "4. Packaging gt-checksum" +cp -rpf Oracle/${OracleDrive}.tar.xz gt-checksum README.md CHANGELOG.md gc-sample.conf gt-checksum-manual.md gt-checksum-${vs}-linux-${arch} +tar zcf gt-checksum-${vs}-linux-${arch}.tar.gz gt-checksum-${vs}-linux-${arch} +echo "5. The gt-checksum binary package is: gt-checksum-${vs}-linux-${arch}.tar.gz under directory release" +mv gt-checksum-${vs}-linux-${arch}.tar.gz release +ls -la release +rm -fr gt-checksum-${vs}-linux-${arch} diff --git a/dbExec/schem_Table_Column.go b/dbExec/schem_Table_Column.go index 168c8f6ea9c1196ec520f984618e13af7487187d..5c06496a7a9b0bef98235eeff267e6dcdf721a05 100644 --- a/dbExec/schem_Table_Column.go +++ b/dbExec/schem_Table_Column.go @@ -13,7 +13,7 @@ type TableColumnNameStruct struct { Drive string Db *sql.DB Datafix string - LowerCaseTableNames string + CaseSensitiveObjectName string } type QueryTableColumnNameInterface interface { @@ -38,7 +38,7 @@ func (tcns *TableColumnNameStruct) Query() QueryTableColumnNameInterface { Schema: tcns.Schema, Table: tcns.Table, Db: tcns.Db, - LowerCaseTableNames: tcns.LowerCaseTableNames, + CaseSensitiveObjectName: tcns.CaseSensitiveObjectName, } } if tcns.Drive == "godror" { @@ -46,7 +46,7 @@ func (tcns *TableColumnNameStruct) Query() QueryTableColumnNameInterface { Schema: tcns.Schema, Table: tcns.Table, Db: tcns.Db, - LowerCaseTableNames: tcns.LowerCaseTableNames, + CaseSensitiveObjectName: tcns.CaseSensitiveObjectName, } } return aa diff --git a/docs/gc.conf.example b/docs/gc.conf.example deleted file mode 100644 index 97d7600fad4e4ea8646fe5247ddd265af9704690..0000000000000000000000000000000000000000 --- a/docs/gc.conf.example +++ /dev/null @@ -1,116 +0,0 @@ -; gt-checksum 配置文件参考 - -; 定义源、目标数据源 -; 目前只支持MySQL、Oracle两种数据源 - -[DSNs] -;oracle的连接串格式为：oracle|user/password@ip:port/sid -;例如：srcDSN = oracle|scott/tiger@172.16.0.1:1521/helowin - -;mysql的连接串格式为：mysql|usr:password@tcp(ip:port)/dbname?charset=xxx -;例如：dstDSN = mysql|pcms:abc123@tcp(172.16.0.1:3306)/information_schema?charset=utf8 - -srcDSN = mysql|pcms:abc123@tcp(172.16.0.1:3306)/information_schema?charset=utf8 -dstDSN = mysql|pcms:abc123@tcp(172.16.0.2:3306)/information_schema?charset=utf8 - -; 定义校验数据对象 -[Schema] -; 配置参数中，table=*.*表示匹配所有库（MySQL不包含 information_schema\mysql\performance_schema\sys），库表都支持模糊匹配（无论是table还是ignoreTable），%代表模糊，*代表所有，包含的模糊规则：%schema.xxx，%schema%.xxx schema%.xxx schema.%table schema.table% schema.%table% schema.table 其中如果设置了*.*，则不能在输入其他的值，例如：*.*,pcms%.*，则是错误的，会报table设置错误，table和ignoreTable的值相同，也会报错 - -; 选项tables用来定义校验数据表对象，支持通配符"%"和"*" -; 例如： -; *.* 表示所有库表对象（MySQL不包含 information_schema\mysql\performance_schema\sys） -; test.* 表示test库下的所有表 -; test.t% 表示test库下所有表名中包含字母"t"开头的表 -; db%.* 表示所有库名中包含字母"db"开头的数据库中的所有表 -; %db.* 表示所有库名中包含字母"db"结尾的数据库中的所有表 -; -; 如果已经设置为 "*.*"，则不能再增加其他的规则，例如：设置 "*.*,pcms%.*" 则会报告规则错误 -; 如果 table 和 ignore-tables 设置的值相同的话也会报告规则错误 -tables = test.* - -; 选项 ignore-tables 用来定义忽略的数据对象规则，也支持通配符"%"和"*"，具体用法参考上面的案例 -; ignore-tables = db1.* -ignore-tables = - -; 设置是否检查没有索引的表，可设置为：yes/no，默认值为：no -; checkNoIndexTable = yes | no -checkNoIndexTable = yes - -; 设置是否忽略表名大小写，可设置为：yes/no，默认值为：no -; yes => 会按照配置的大小写进行匹配 -; no => 统一用大写表名 -; lowerCaseTableNames = yes | no -lowerCaseTableNames = no - -; 设置日志文件名及等级 -[Logs] -; 设置日志文件名，可以指定为绝对路径或相对路径 -log = ./gt-checksum.log - -; 设置日志等级，支持 debug/info/warn/error 几个等级，默认值为：info -; logLevel = info | debug | warn | error -logLevel = info - -; 其他校验规则 -[Rules] -; 数据校验并行线程数 -parallel-thds = 10 - -; 设置单列索引每次检索多少条数据进行校验，默认值：1000，建议范围：1000 - 5000 -; 注：该值设置太大时有可能会造成SQL查询效率反倒下降的情况发生，一般建议设置不超过5000 -singleIndexChanRowCount = 10000 - -; 设置多列索引每次检索多少条数据进行校验，默认值：1000，建议范围：1000 - 5000 -; 注：该值设置太大时有可能会造成SQL查询效率反倒下降的情况发生，一般建议设置不超过5000 -jointIndexChanRowCount = 10000 - -; 设置校验队列深度，默认值：100 -queue-size = 100 - -; 设置数据校验模式，支持 count/rows/sample 三种模式，默认值为：rows -; count 表示只校验源、目标表的数据量 -; rows 表示逐行校验源、目标数据 -; sample 表示只进行抽样数据校验，配合参数ratio设置采样率 -; checkMode = rows | count | sample -checkMode = rows - -; 当 checkMode = sample 时，设置数据采样率，设置范围1-100，用百分比表示，1表示1%，100表示100%，默认值：10 -; ratio = 10 - -; 设置数据校验对象，支持 data/struct/index/partitions/foreign/trigger/func/proc，默认值为：data -; 分别表示：行数据/表结构/索引/分区/外键/触发器/存储函数/存储过程 -; checkObject = data | struct | index | partitions | foreign | trigger | func | proc -checkObject = data - -;设置表结构校验规则，当checkObject为struct时才会生效 -[Struct] -; 设置struct校验时的校验模式，可设置为：strict/loose，为strict时，则会严格匹配列的所有属性，为loose时，则为宽松模式只匹配列名，默认值为：strict -; ScheckMod = strict | loose -ScheckMod = strict - -; 设置struct校验时是否校验列的顺序，可设置为：yes/no，设置为yes，则会按照源端的列的正序进行校验，默认值为：yes -; ScheckOrder = yes | no -ScheckOrder = yes - -; 设置修复列的属性及顺序的依据原则，可设置为src/dst，设置为src则按照源端的列属性进行修复，默认值为：src -; 当缺少列时，修复语句会按照源端的列数据类型生成 -; ScheckFixRule = src | dst -ScheckFixRule = src - -; 设置日志文件名及等级 - -; 设置数据修复方案 -[Repair] -; 数据修复方式，支持 file/table 两种方式 -; file，生成数据修复SQL文件 -; table 直接在线修复数据 -; datafix = file | table -datafix = file - -; 修复事务数，即单个事务包含多少个dml语句，默认值为：100 -fixTrxNum = 100 - -; 当 datafix = file 时，设置生成的SQL文件名，可以指定为绝对路径或相对路径 -; 当 datafix = table 时，可以不用设置 fixFileName 参数 -fixFileName = ./gt-checksum-DataFix.sql diff --git a/docs/gt-checksum-manual.md b/docs/gt-checksum-manual.md deleted file mode 100644 index 18fae92ddfe16612ad0a629093c8255218a5ea64..0000000000000000000000000000000000000000 --- a/docs/gt-checksum-manual.md +++ /dev/null @@ -1,499 +0,0 @@ -# gt-checksum ---- - -## 关于gt-checksum ---- -gt-checksum - A opensource table and data checksum tool by GreatSQL - -## 用法 ---- -Usage: - -``` -gt-checksum --srcDSN DSN --dstDSN DSN --tables TABLES -``` - -or - -``` -gt-checksum --config=./gc.conf -``` - -### 数据库授权 -想要运行gt-checksum工具，需要至少授予以下几个权限： -- 在MySQL端 - - 1.全局权限 - - a.`REPLICATION CLIENT` - b.`SESSION_VARIABLES_ADMIN`，如果是MySQL 8.0版本的话，MySQL 5.7版本不做这个要求 - - 2.校验数据对象 - - a.如果`datafix=file`，则只需要`SELECT`权限 - - b.如果`datafix=table`，则需要`SELECT、INSERT、DELETE`权限，如果还需要修复表结构不一致的情况，则需要`ALTER`权限 - -假设现在要对db1.t1做校验和修复，则可授权如下 - -``` -mysql> GRANT REPLICATION CLIENT, SESSION_VARIABLES_ADMIN ON *.* to ...; -mysql> GRANT SELECT, INSERT, DELETE ON db1.t1 to ...; -``` -- 在Oracle端 - - 1.全局权限 - - a.`SELECT ANY DICTIONARY` - - 2.校验数据对象 - - a.如果`datafix=file`，则只需要`SELECT ANY TABLE`权限 - - b.如果`datafix=table`，则需要`SELECT ANY TABLE、INSERT ANY TABLE、DELETE ANY TABLE`权限 - - -### 快速使用案例1 ---- -指定配置文件，开始执行数据校验，示例： -```shell -shell> ./gt-checksum -f ./gc.conf --- gt-checksum init configuration files -- --- gt-checksum init log files -- --- gt-checksum init check parameter -- --- gt-checksum init check table name -- --- gt-checksum init check table column -- --- gt-checksum init check table index column -- --- gt-checksum init source and dest transaction snapshoot conn pool -- --- gt-checksum init cehck table query plan and check data -- -begin checkSum index table db1.t1 -[█████████████████████████████████████████████████████████████████████████████████████████████████████████████████]113% task: 678/600 -table db1.t1 checksum complete - -** gt-checksum Overview of results ** -Check time: 73.81s (Seconds) -Schema Table IndexCol checkMod Rows Differences Datafix -db1 t1 ol_w_id,ol_d_id,ol_o_id,ol_number rows 5995934,5995918 yes file -``` - -### 快速使用案例2 ---- -设定只校验db1库下的所有表，不校验test库下的所有表，并设置没有索引的表也要校验 -``` -./gt-checksum -S type=mysql,user=root,passwd=abc123,host=172.16.0.1,port=3306,charset=utf8 -D type=mysql,user=root,passwd=abc123,host=172.16.0.2,port=3306,charset=utf8 -t db1.* -it test.* -nit yes -``` - -## gt-checksum特性 ---- -MySQL DBA最常用的数据校验&修复工具应该是Percona Toolkit中的pt-table-checksum和pt-table-sync这两个工具，不过这两个工具并不支持MySQL MGR架构，以及国内常见的上云下云业务场景，还有MySQL、Oracle间的异构数据库等多种场景。 - -GreatSQL开源的gt-checksum工具可以满足上述多种业务需求场景，解决这些痛点。 - -gt-checksum工具支持以下几种常见业务需求场景： -1. **MySQL主从复制**：主从复制中断后较长时间才发现，且主从间差异的数据量太多，这时候通常基本上只能重建复制从库，如果利用pt-table-checksum先校验主从数据一致性后，再利用pt-table-sync工具修复差异数据，这个过程要特别久，时间代价太大。 -2. **MySQL MGR组复制**：MySQL MGR因故崩溃整个集群报错退出，或某个节点异常退出，在恢复MGR集群时一般要面临着先检查各节点间数据一致性的需求，这时通常为了省事会选择其中一个节点作为主节点，其余从节点直接复制数据重建，这个过程要特别久，时间代价大。 -3. **上云下云业务场景**：目前上云下云的业务需求很多，在这个过程中要进行大量的数据迁移及校验工作，如果出现字符集改变导致特殊数据出现乱码或其他的情况，如果数据迁移工具在迁移过程中出现bug或者数据异常而又迁移成功，此时都需要在迁移结束后进行一次数据校验才放心。 -4. **异构迁移场景**：有时我们会遇到异构数据迁移场景，例如从Oracle迁移到MySQL，通常存在字符集不同，以及数据类型不同等情况，也需要在迁移结束后进行一次数据校验才放心。 -5. **定期校验场景**：作为DBA在维护高可用架构中为了保证主节点出现异常后能够快速放心切换，就需要保证各节点间的数据一致性，需要定期执行数据校验工作。 - -以上这些场景，都可以利用gt-chcksum工具来满足。 - - -## 参数选项详解 ---- -gt-checksum支持命令行传参，或者指定配置文件两种方式运行，但不支持两种方式同时指定。 - -配置文件可参考(这个模板)[x]，模板中包含相关参数的详细解释。 - -gt-checksum命令行参数选项详细解释如下： - -- --config / -f - Type: string - - 指定配置文件名，例如： - -```shell -shell> ./gt-checksum -f ./gc.conf -shell> ./gt-checksum --config ./gc.conf -``` -gt-checksum支持极简配置文件工作方式，即只需要最少的几个参数就能工作，例如： -```shell -# -shell> cat gc.conf-simple -[DSNs] -srcDSN = mysql|pcms:abc123@tcp(172.17.16.1:3306)/information_schema?charset=utf8 -dstDSN = mysql|pcms:abc123@tcp(172.17.16.2:3306)/information_schema?charset=utf8 - -[Schema] -tables = db1.t1 -``` -**注意**： - -1. 极简配置文件名必须是 `gc.conf-simple`。 -2. 配置文件中仅需指定源和目标端的DSN，以及要校验的表名即可。 - -- --srcDSN / -S - Type: String. Default: port=3306,charset=utf8mb4. - - 定义数据校验源数据库的DSN，例如： -``` - -S type=mysql,user=root,passwd=abc123,host=172.17.140.47,port=3306,charset=utf8mb4 -``` - 当前DSN定义支持MySQL、Oracle两种数据库。 - - Oracle的连接串格式为：`oracle|user/password@ip:port/sid` - 例如：`srcDSN = oracle|pcms/abc123@172.16.0.1:1521/helowin` - - MySQL的连接串格式为：`mysql|usr:password@tcp(ip:port)/dbname?charset=xxx` - 例如：`dstDSN = mysql|pcms:abc123@tcp(172.16.0.1:3306)/information_schema?charset=utf8` - - 注：port默认值是3306，charset默认值是utf8mb4。 - -- --dstDSN / -D - Type: String. Default: port=3306,charset=utf8mb4. - - 定义数据校验目标数据库的DSN，例如： - -``` --D type=mysql,user=root,passwd=abc123,host=172.17.140.47,port=3306,charset=utf8mb4 -``` - 和srcDSN一样，也支持MySQL、Oracle两种数据库，DSN字符串格式同srcDSN。 - - 注：port默认值是3306，charset默认值是utf8mb4。 - -- --table / -t - Type: String. Default: nil. - - 定义要执行数据校验的数据表对象列表，支持通配符"%"和"*"。 - - 表名中支持的字符有：[0-9 a-z! @ _ {} -]. [0-9 a-z! @ _ {} -]，超出这些范围的表名将无法识别。 - - 下面是几个案例： - - *.* 表示所有库表对象（MySQL不包含 information_schema\mysql\performance_schema\sys） - - test.* 表示test库下的所有表 - - test.t% 表示test库下所有表名中包含字母"t"开头的表 - - db%.* 表示所有库名中包含字母"db"开头的数据库中的所有表 - - %db.* 表示所有库名中包含字母"db"结尾的数据库中的所有表 - - 如果已经设置为 "*.*"，则不能再增加其他的规则，例如：设置 "*.*,pcms%.*" 则会报告规则错误。 如果 table 和 ignore-tables 设置的值相同的话也会报告规则错误。 - - 案例： -```shell -shell> gt-checksum -S srcDSN -D dstDSN -t db1.* -``` - -- --ignore-table / -it - Type: String. Default: nil. - - 定义不要执行数据校验的数据表对象列表，支持通配符"%"和"*"。 - - 表名中支持的字符有：[0-9 a-z! @ _ {} -]. [0-9 a-z! @ _ {} -]，超出这些范围的表名将无法识别。 - - 具体用法参考上面 --table 选项中的案例。 - - 案例： -```shell -shell> gt-checksum -S srcDSN -D dstDSN -t db1.* -it test.* -``` - -- --CheckNoIndexTable / -nit - Type: Bool, yes/no. Default: no. - - 设置是否检查没有索引的表，可设置为：yes/no，默认值为：no。 - - 当设置为yes时，会对没有索引的表也执行数据校验，这个校验过程可能会非常慢。 - - 案例： -```shell -shell> gt-checksum -S srcDSN -D dstDSN -t db1.* -nit yes -``` - -- --lowerCase / -lc - Type: Bool, yes/no. Default: no. - - 设置是否忽略表名大小写，设置为：yes/no，默认值为：no。 - - yes => 会按照配置的大小写进行匹配；no => 统一用大写表名。 - - 案例： -```shell -shell> gt-checksum -S srcDSN -D dstDSN -t db1.* -lc no -``` - -- --logFile / -lf - Type: String. Default: ./gt-checksum.log. - - 设置日志文件名，可以指定为绝对路径或相对路径。 - -./gt-checksum -S DSN -D DSN -lf gt-checksum.log - - 案例： -```shell -shell> gt-checksum -S srcDSN -D dstDSN -t db1.* -lf ./gt-checksum.log -``` - -- --logLevel, -ll - Type: String, debug/info/warn/error. Default: info. - - 设置日志等级，支持 debug/info/warn/error 几个等级，默认值为：info。 - - 案例： -```shell -shell> gt-checksum -S srcDSN -D dstDSN -t db1.* -lf ./gt-checksum.log -ll info -``` - -- --parallel-thds / -thds - Type: Int. Default: 5. - - 设置数据校验并行线程数。该值必须设置大于0，并行线程数越高，数据校验速度越快，系统负载也会越高，网络连接通信也可能会成为瓶颈。 - - 案例： -```shell -shell> gt-checksum -S srcDSN -D dstDSN -t db1.* -thds 5 -``` - -- --singleIndexChanRowCount / -sicr - Type: Int. Default: 1000. - - 设置单列索引每次检索多少条数据进行校验，默认值：1000，建议范围：1000 - 5000。 - - 注：该值设置太大时有可能会造成SQL查询效率反倒下降的情况发生，一般建议设置不超过5000。 - - 案例： -```shell -./gt-checksum -S DSN -D DSN -t db1.* -sicr 1000 -``` - -- --jointIndexChanRowCount / -jicr - Type: Int. Default: 1000. - - 设置多列索引每次检索多少条数据进行校验，默认值：1000，建议范围：1000 - 5000。 - - 注：该值设置太大时有可能会造成SQL查询效率反倒下降的情况发生，一般建议设置不超过5000。 - - 案例： -```shell -./gt-checksum -S DSN -D DSN -t db1.* -jicr 1000 -``` - -- --queue-size / -qs - Type: Int. Default: 100. - - 设置数据校验队列深度，默认值：100。 - - 数据校验队列深度值设置越大，需要消耗的内存会越高，校验的速度也会越快。 - - 案例： -```shell -./gt-checksum -S DSN -D DSN -t db1.* -qs 100 -``` - -- --checkMode / -cm - Type: enum, count/rows/sample. Default: rows. - - 设置数据校验模式，支持 count/rows/sample 三种模式，默认值为：rows - - count 表示只校验源、目标表的数据量 - - rows 表示逐行校验源、目标数据 - - sample 表示只进行抽样数据校验，配合参数ratio设置采样率 - - 案例： -```shell -./gt-checksum -S DSN -D DSN -t db1.* -cm rows -``` - -- --ratio / -r - Type: Int. Default: 10. - - 当 `checkMode = sample` 时，设置数据采样率，设置范围1-100，用百分比表示，1表示1%，100表示100%，默认值：10。 - - 案例： -```shell -./gt-checksum -S DSN -D DSN -t db1.* -cm sample -r 10 -``` - -- --checkObject / -co - Type: enum, data/struct/index/partitions/foreign/trigger/func/proc. Default: data. - - 设置数据校验对象，支持 data/struct/index/partitions/foreign/trigger/func/proc，默认值为：data - - 分别表示：行数据/表结构/索引/分区/外键/触发器/存储函数/存储过程。 - - 案例： -```shell -./gt-checksum -S DSN -D DSN -t db1.* -co data -``` -- --ScheckFixRule value, --sfr value column to fix based on. For example: --sfr src (default: "src") [$src, $dst] - Type: enum, src/dst - - 设置表结构校验时，数据修复时的对准原则，选择源端 或 目标端作为数据修复的依据。 - - 案例 -```shell -./gt-checksum -S DSN -D DSN -t db1.* -sfr=src -``` - -- --ScheckOrder value, --sco value The positive sequence check of column. For example: --sco yes (default: "yes") [$yes, $no] - Type: Bool, yes/no. Default: no. - - 设置表结构数据校验时，是否要检查数据列的顺序。 - - 案例： -```shell -./gt-checksum -S DSN -D DSN -t db1.* -sco=yes -``` - -- --ScheckMod value, --scm value column check mode. For example: --scm strict (default: "strict") [$strict, $loose] - Type: enum, strict/loose - - 设置表结构校验时采用严格还是宽松模式。 - - 宽松模式，只匹配数据列名。 - - 严格模式，严格匹配数据列的属性，列的属性包括数据类型、是否允许为null、默认值、字符集、校验集、comment等。 - - 案例： -```shell -./gt-checksum -S DSN -D DSN -t db1.* -scm=strict -``` - -- --datafix / -df - Type: enum, table/file. Default: file. - - 设置数据修复方式，支持 file/table 两种方式。file：生成数据修复SQL文件；table：直接在线修复数据。 - - 案例： -```shell -./gt-checksum -S DSN -D DSN -t db1.* -df file -or -./gt-checksum -S DSN -D DSN -t db1.* -df table -``` - -- --fixFileName / -ffn - Type: String. Default: ./gt-checksum-DataFix.sql - - 当 datafix = file 时，设置生成的SQL文件名，可以指定为绝对路径或相对路径。 - - 当 datafix = table 时，可以不用设置 fixFileName 参数。 - -./gt-checksum -S DSN -D DSN -ffn gt-checksum-DataFix.sql - - 案例： -```shell -./gt-checksum -S DSN -D DSN -t db1.* -df file -ffn ./gt-checksumDataFix.sql -``` -- --fixTrxNum / -ftn - Type: Int. Default: 100. - - 设置执行数据修复时一个事务中最多运行多少条SQL，或者生成数据修复的SQL文件时，显式在SQL文件中添加 begin + commit 事务起止符中间的SQL语句数量。 - - 案例： -```shell -./gt-checksum -S DSN -D DSN -t db1.* -ftn=100 -``` - -- --help / -h - 查看帮助内容。 - -- --version / -v - 打印版本号。 - -## 下载 ---- -可以 [这里](https://gitee.com/GreatSQL/gt-checksum/releases) 下载预编译好的二进制文件包，已经在Ubuntu、CentOS、RHEL等多个下测试通过。 - -如果需要校验Oracle数据库，则还需要先下载Oracle数据库相应版本的驱动程序，并配置驱动程序使之生效。例如：待校验的数据库为Oracle 11-2，则要下载Oracle 11-2的驱动程序，并使之生效，否则连接Oracle会报错。详细方法请见下方内容：[**下载配置Oracle驱动程序**](x) 。 - - -## 下载配置Oracle驱动程序 ---- -如果需要校验Oracle数据库，则还需要先下载Oracle数据库相应版本的驱动程序。例如：待校验的数据库为Oracle 11-2，则要下载Oracle 11-2的驱动程序，并使之生效，否则连接Oracle会报错。 - -### 下载Oracle Instant Client -从 [https://www.oracle.com/database/technologies/instant-client/downloads.html](https://www.oracle.com/database/technologies/instant-client/downloads.html) 下载免费的Basic或Basic Light软件包。 - -- oracle basic client, instantclient-basic-linux.x64-11.2.0.4.0.zip - -- oracle sqlplus, instantclient-sqlplus-linux.x64-11.2.0.4.0.zip - -- oracle sdk, instantclient-sdk-linux.x64-11.2.0.4.0.zip - -### 配置oracle client并生效 -```shell -shell> unzip instantclient-basic-linux.x64-11.2.0.4.0.zip -shell> unzip instantclient-sqlplus-linux.x64-11.2.0.4.0.zip -shell> unzip instantclient-sdk-linux.x64-11.2.0.4.0.zip -shell> mv instantclient_11_2 /usr/local -shell> echo "export LD_LIBRARY_PATH=/usr/local/instantclient_11_2:$LD_LIBRARY_PATH" >> /etc/profile -shell> source /etc/profile -``` - -## 源码编译 -gt-checksum工具采用GO语言开发，您可以自行编译生成二进制文件。 - -编译环境要求使用golang 1.17及以上版本。 - -请参考下面方法下载源码并进行编译： -```shell -shell> git clone https://gitee.com/GreatSQL/gt-checksum.git -shell> go build -o gt-checksum gt-checksum.go -shell> chmod +x gt-checksum -shell> mv gt-checksum /usr/local/bin -``` - -## 已知缺陷 -截止最新的1.2.1版本中，当表中有多行数据是完全重复的话，可能会导致校验结果不准确。 - -源端有个表t1，表结构及数据如下： -``` -mysql> show create table t1\G -*************************** 1. row *************************** - Table: t1 -Create Table: CREATE TABLE `t1` ( - `id` float(10,2) DEFAULT NULL, - `code` varchar(10) DEFAULT NULL, - KEY `idx_1` (`id`,`code`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -mysql> select * from t1; -+-------+------+ -| id | code | -+-------+------+ -| 1.01 | a | -| 1.50 | b | -| 2.30 | c | -| 3.40 | d | -| 4.30 | NULL | -| 4.30 | NULL | -| 4.30 | NULL | -| 4.30 | | -| 4.30 | f | -| 10.10 | e | -+-------+------+ -10 rows in set (0.00 sec) -``` -**注意**：上述10行数据中，有3行数据是完全一致的。 - -目标端中同样也有t1表，表结构完全一样，但数据不一样： -``` -mysql> select * from t1; -+-------+------+ -| id | code | -+-------+------+ -| 1.01 | a | -| 1.50 | b | -| 2.30 | c | -| 3.40 | d | -| 4.30 | NULL | -| 4.30 | | -| 4.30 | f | -| 10.10 | e | -+-------+------+ -8 rows in set (0.00 sec) -``` - -可以看到，目标端中的t1表只有8行数据，如果除去重复数据，两个表是一致的，这也会导致校验的结果显示为一致。 -``` -... -** gt-checksum Overview of results ** -Check time: 0.30s (Seconds) -Schema Table IndexCol checkMod Rows Differences Datafix -t1 T1 id,code rows 10,8 no file -``` -这个问题我们会在未来某个版本中尽快修复。 - -## BUGS ---- -可以 [戳此](https://gitee.com/GreatSQL/gt-checksum/issues) 查看 gt-checksum 相关bug列表。 diff --git a/gc-sample.conf b/gc-sample.conf new file mode 100644 index 0000000000000000000000000000000000000000..28ab94205d05d48d14fef7242a052eccc7d059c1 --- /dev/null +++ b/gc-sample.conf @@ -0,0 +1,166 @@ +; gt-checksum 配置文件参考 + +; 定义源、目标数据源 +; 这部分参数不能全部为空 +[DSNs] +srcDSN = mysql|u1:p1@tcp(172.17.0.1:3307)/information_schema?charset=utf8mb4 +dstDSN = mysql|u1:p1@tcp(172.17.0.2:3307)/information_schema?charset=utf8mb4 +; 目前只支持MySQL、Oracle两种数据源 +; Oracle DSN格式为：oracle|user/password@ip:port/sid +; 例如：srcDSN = oracle|scott/tiger@172.16.0.1:1521/helowin +; +; MySQL DSN格式为：mysql|usr:password@tcp(ip:port)/dbname?charset=xxx +; 例如：dstDSN = mysql|checksum:Checksum@3306@tcp(172.16.0.1:3306)/information_schema?charset=utf8mb4 + + +; 定义校验数据对象 +; 这部分参数不能全部为空，至少要配置tables参数 +[Schema] +tables = test.* +; 配置参数中，table=*.*表示匹配所有库（MySQL数据源则自动排除 information_schema, mysql, performance_schema, sys 等几个系统库） +; 库表名称都支持模糊匹配（无论是table还是ignoreTable） +; %代表模糊，*代表所有 +; 常用模糊规则示例： +; - %schema.xxx +; - %schema%.xxx +; - schema%.xxx +; - schema.%table +; - schema.table% +; - schema.%table% +; - schema.table +; 如果已经设置了"*.*"，则不能再添加其他值 +; 例如设置规则"*.*,pcms%.*"时会报告table参数设置错误 +; 当参数table和ignoreTable的值设置相同时也会报错 + +; 参数tables用来定义校验数据表对象，支持通配符"%"和"*" +; 例如： +; *.* 表示所有库表对象（MySQL数据源则自动排除 information_schema, mysql, performance_schema, sys 等几个系统库） +; test.* 表示test库下的所有表 +; test.t% 表示test库下所有表名中包含字母"t"开头的表 +; db%.* 表示所有库名中包含字母"db"开头的数据库中的所有表 +; %db.* 表示所有库名中包含字母"db"结尾的数据库中的所有表 + +ignoreTables = +; 参数 ignoreTables 用来定义忽略的数据对象规则 +; ignoreTables 参数的配置规则和 tables 参数相同 + +checkNoIndexTable = no +; 设置是否检查没有索引的表，可设置为 [yes | no]，默认值：no +; 如果为设置则使用默认值 no +; checkNoIndexTable = yes | no + +caseSensitiveObjectName = no +; 设置是否忽略库名、表名、字段名等数据对象名的大小写，可设置为 [yes | no]，默认值：no +; no，忽略tables和ignoreTables涉及的数据对象名大小写规则，统一使用大写库名及表名 +; yes，按照tables和ignoreTables参数涉及的数据对象名的大小写进行匹配 +; caseSensitiveObjectName = yes | no + + +; 其他校验规则 +; 这部分参数如果不设置则使用相应的默认值 +[Rules] +parallelThds = 10 +; 数据校验工作时的并行线程数，默认值：10 + +chunkSize = 10000 +; 设置单列索引每次检索多少条数据进行校验，默认值：10000，建议范围：1000 - 10000 +; 注：该值设置太大时有可能会造成SQL查询效率反倒下降的情况发生，一般建议设置不超过10000 + +queueSize = 1000 +; 设置校验队列深度，默认值：1000 + +checkMode = rows +; 设置数据校验模式，可设置为 [count|rows|sample]，默认值：rows +; count 表示只校验源、目标表的数据量 +; rows 表示逐行校验源、目标数据 +; sample 表示只进行抽样数据校验，配合参数ratio设置采样率 +; checkMode = rows | count | sample + +ratio = 10 +; 当 checkMode = sample 时，设置数据采样率，默认值：10 +; 设置范围 [1-100]，用百分比表示，1表示1%，100表示100% + +checkObject = data +; 设置数据校验对象，可设置为 [data|struct|index|partitions|foreign|trigger|func|proc]，默认值：data +; 分别表示：行数据|表结构|索引|分区|外键|触发器|存储函数|存储过程 +; checkObject = data | struct | index | partitions | foreign | trigger | func | proc + +memoryLimit = 1024 +; 设置内存使用限制，当内存使用超过该值时，会自动退出，避免内存溢出 +; 设置范围 [100 - 65536]，默认值：1024，单位：MB（不支持附加KB、MB、GB等单位写法） + + +; 设置表结构校验规则，当checkObject为struct时才会生效 +; 这部分参数如果不设置则使用相应的默认值 +[Struct] +ScheckMod = strict +; 设置struct校验时的校验模式，可设置为：[strict | loose]，默认值：strict +; strict，严格匹配列的所有属性 +; loose，宽松模式，只匹配列名 +; ScheckMod = strict | loose + +ScheckOrder = yes +; 设置struct校验工作模式下时是否校验列的顺序，可设置为：[yes | no]，默认值：yes +; yes，按照源端的列的正序进行校验 +; no，无需按照源端的列的正序进行检查 +; ScheckOrder = yes | no + +ScheckFixRule = src +; 设置修复列的属性及顺序的依据原则，可设置为 [src | dst]，默认值：src +; src，按照源端的列属性进行修复 +; dst，按照目标端的列属性进行修复 +; 当缺少列时，修复语句会按照源端的列数据类型生成 +; ScheckFixRule = src | dst + + +; 设置数据修复方案 +; 这部分参数如果不设置则使用相应的默认值 +[Repair] +datafix = file +; 数据修复方式，可设置为 [file | table] +; file，生成数据修复SQL文件 +; table，直接在线修复数据 +; datafix = file | table + +fixTrxNum = 100 +; 修复事务数，即单个事务包含多少个DML语句，默认值：100 + +fixFileName = ./gt-checksum-DataFix.sql +; 设置生成修复SQL文件时相应的文件名 +; 当 datafix = file 时，设置生成的SQL文件名，可以指定为绝对路径或相对路径 +; 当 datafix = table 时，可以不用设置 fixFileName 参数 + + +; 设置日志文件名及等级 +; 这部分参数如果不设置则使用相应的默认值 +[Logs] +logFile = ./gt-checksum.log +; 设置日志文件名，可以指定为绝对路径或相对路径 +; 如果不设置则使用默认值 "./gt-checksum.log" +; 支持日期时间格式，例如：gt-checksum-%Y%m%d%H%M%S.log +; 当前可支持的日期时间格式有以下几种： +; %Y：年（如2025） +; %m：月（如09） +; %d：日（如09） +; %H：小时（如08） +; %M：分钟（如08） +; %S：秒（如08） +; %s：当前时间戳（如1757409298） +; %F：完整日期（如2025-09-09） +; %T：完整时间（如08:08:08） + +logLevel = info +; 设置日志等级，可设置为 [debug | info | warn | error]，默认值：info +; 如果不设置则使用默认值 info + + +; +; 一个极简配置文件案例 +; 仅需配置srcDSN, dstDSN, tables三个参数，其余参数均自动使用默认值 +; +;[DSNs] +;srcDSN = mysql|checksum:checksum@tcp(127.0.0.1:3306)/information_schema?charset=utf8mb4 +;dstDSN = mysql|checksum:checksum@tcp(127.0.0.1:3307)/information_schema?charset=utf8mb4 +; +;[Schema] +;tables=sbtest.sbtest1 diff --git a/gc.conf b/gc.conf deleted file mode 100644 index 13eeac34d74c68d1445db678d240053f767ef69c..0000000000000000000000000000000000000000 --- a/gc.conf +++ /dev/null @@ -1,94 +0,0 @@ -; gt-checksum 配置文件参考 - -; 定义源、目标数据源 -; 目前只支持MySQL、Oracle两种数据源 - -[DSNs] -;oracle的连接串格式为：oracle|user/password@ip:port/sid -;例如：srcDSN = oracle|scott/tiger@172.16.0.1:1521/helowin - -;mysql的连接串格式为：mysql|usr:password@tcp(ip:port)/dbname?charset=xxx -srcDSN = mysql|u1:p1@tcp(172.17.0.1:3307)/information_schema?charset=utf8mb4 -dstDSN = mysql|u1:p1@tcp(172.17.0.2:3307)/information_schema?charset=utf8mb4 - -; 定义校验数据对象 -[Schema] -; 选项tables用来定义校验数据表对象，支持通配符"%"和"*" -; 例如： -; *.* 表示所有库表对象（MySQL不包含 information_schema\mysql\performance_schema\sys） -; test.* 表示test库下的所有表 -; test.t% 表示test库下所有表名中包含字母"t"开头的表 -; db%.* 表示所有库名中包含字母"db"开头的数据库中的所有表 -; %db.* 表示所有库名中包含字母"db"结尾的数据库中的所有表 -; -; 如果已经设置为 "*.*"，则不能再增加其他的规则，例如：设置 "*.*,pcms%.*" 则会报告规则错误 -; 如果 table 和 ignore-tables 设置的值相同的话也会报告规则错误 - -tables = db1.t1 -ignore-tables = - -; 设置是否检查没有索引的表，可设置为：yes/no，默认值为：no -checkNoIndexTable = no - -; 设置是否忽略表名大小写，可设置为：yes/no -; 当为no时，统一使用大写表名；当为yes时，会按照配置的大小写进行匹配 -; 默认值为：no -lowerCaseTableNames = no - -; 其他校验规则 -[Rules] -; 数据校验并行线程数 -parallel-thds = 10 - -; 设置每次检索多少条数据进行校验，默认值：10000 -chanRowCount = 10000 - -; 设置校验队列深度，默认值：100 -queue-size = 100 - -; 设置数据校验模式，支持 count/rows/sample 三种模式，默认值为：rows -; count 表示只校验源、目标表的数据量 -; rows 表示逐行校验源、目标数据 -; sample 表示只进行抽样数据校验，配合参数ratio设置采样率 -checkMode = rows - -; 当 checkMode = sample 时，设置数据采样率，设置范围1-100，用百分比表示，1表示1%，100表示100%，默认值：10 -ratio = 10 - -; 设置数据校验对象，支持 data/struct/index/partitions/foreign/trigger/func/proc，默认值为：data -; 分别表示：行数据/表结构/索引/分区/外键/触发器/存储函数/存储过程 -checkObject = data - -;设置表结构校验规则，当checkObject为struct时才会生效 -[Struct] -; 设置struct校验时的校验模式，可设置为：strict/loose，为strict时，则会严格匹配列的所有属性，为loose时，则为宽松模式只匹配列名，默认值为：strict -ScheckMod = strict - -; 设置struct校验时是否校验列的顺序，可设置为：yes/no，设置为yes，则会按照源端的列的正序进行校验，默认值为：yes -ScheckOrder = yes - -; 设置修复列的属性及顺序的依据原则，可设置为src/dst，设置为src则按照源端的列属性进行修复，默认值为：src -; 当缺少列时，修复语句会按照源端的列数据类型生成 -ScheckFixRule = src - -; 设置日志文件名及等级 -[Logs] -; 设置日志文件名，可以指定为绝对路径或相对路径 -log = ./gt-checksum.log - -; 设置日志等级，支持 debug/info/warn/error 几个等级，默认值为：info -logLevel = info - -; 设置数据修复方案 -[Repair] -; 数据修复方式，支持 file/table 两种方式 -; file，生成数据修复SQL文件 -; table 直接在线修复数据 -datafix = file - -; 修复事务数，即单个事务包含多少个dml语句，默认值为：100 -fixTrxNum = 100 - -; 当 datafix = file 时，设置生成的SQL文件名，可以指定为绝对路径或相对路径 -; 当 datafix = table 时，可以不用设置 fixFileName 参数 -fixFileName = ./gt-checksum-DataFix.sql diff --git a/gc.conf-simple b/gc.conf-simple deleted file mode 100644 index 1c5ffeb9d4f2805e60f58fbae63decff4a85cd66..0000000000000000000000000000000000000000 --- a/gc.conf-simple +++ /dev/null @@ -1,23 +0,0 @@ -; -; gc.cnf-simple -; -; 极简配置文件模板，只需要最少的几个参数即可 -; -[DSNs] -;oracle的连接串为 oracle|user/password@ip:port/sid -;mysql的连接串为 mysql|usr:password@tcp(ip:port)/dbname?charset=xxx -srcDSN = mysql|u1:p1@tcp(172.17.0.1:3307)/information_schema?charset=utf8mb4 -dstDSN = mysql|u1:p1@tcp(172.17.0.2:3307)/information_schema?charset=utf8mb4 - -[Schema] -; 选项tables用来定义校验数据表对象，支持通配符"%"和"*" -; 例如： -; *.* 表示所有库表对象（MySQL不包含 information_schema\mysql\performance_schema\sys） -; test.* 表示test库下的所有表 -; test.t% 表示test库下所有表名中包含字母"t"开头的表 -; db%.* 表示所有库名中包含字母"db"开头的数据库中的所有表 -; %db.* 表示所有库名中包含字母"db"结尾的数据库中的所有表 -; -; 如果已经设置为 "*.*"，则不能再增加其他的规则，例如：设置 "*.*,pcms%.*" 则会报告规则错误 -; 如果 table 和 ignore-tables 设置的值相同的话也会报告规则错误 -tables = db1.t1 \ No newline at end of file diff --git a/go.mod b/go.mod index 9d264f7d09fff956ceb122bab9c16869b968d43b..170f0f973fea0735bec923cadb102a9b966303fd 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 github.com/satori/go.uuid v1.2.0 + github.com/shirou/gopsutil/v3 v3.23.7 github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 github.com/sirupsen/logrus v1.8.1 ) @@ -40,7 +41,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.16.0 // indirect - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.3.6 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index a99f730cc5b9152c102fa1f03dce6b13f8de86f1..e9cbf5ad540bb37d0fb15b01cd390940fffcd6ed 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,7 @@ github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNV github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -32,6 +33,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= @@ -54,6 +56,7 @@ github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECae github.com/lestrrat-go/strftime v1.0.5 h1:A7H3tT8DhTz8u65w+JRpiBxM4dINQhUXAZnhBa2xeOE= github.com/lestrrat-go/strftime v1.0.5/go.mod h1:E1nN3pCbtMSu1yjSVeyuRFVm/U0xoR76fd03sz+Qz4g= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -80,6 +83,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= @@ -90,6 +94,9 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= @@ -103,13 +110,21 @@ github.com/siddontang/go-mysql v1.4.0/go.mod h1:3lFZKf7l95Qo70+3XB2WpiSf9wu2s3na github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/urfave/cli v1.22.9 h1:cv3/KhXGBGjEXLC4bH0sLuJ9BewaAbpk5oyMOveu4pw= github.com/urfave/cli v1.22.9/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= @@ -142,13 +157,18 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -183,5 +203,7 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXL gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/gt-checksum-manual.md b/gt-checksum-manual.md new file mode 100644 index 0000000000000000000000000000000000000000..f6347a4b217db675fa70df06123cb5833778f901 --- /dev/null +++ b/gt-checksum-manual.md @@ -0,0 +1,222 @@ +# gt-checksum 手册 + +## 关于gt-checksum + +**gt-checksum** 是GreatSQL社区开源的数据库校验及修复工具，支持MySQL、Oracle等主流数据库。 + +## 用法 + +指定完整配置文件方式运行 + +```bash +$ gt-checksum -c ./gc.conf +``` + +## 数据库授权 + +运行 gt-checksum 工具前，建议创建相应的专属数据库账户，并至少授予以下几个权限。 + +- MySQL端 + + 1.创建专属账户 + + 执行下面的SQL命令，创建专属账户： + + ```sql + CREATE USER 'checksum'@'%' IDENTIFIED WITH mysql_native_password BY 'Checksum@3306'; + ``` + + 2.全局权限 + + 如果是MySQL 8.0及以上版本，需授予 `REPLICATION CLIENT` 和 `SESSION_VARIABLES_ADMIN` 权限。如果MySQL 5.7级以下版本，则无需授予 `SESSION_VARIABLES_ADMIN` 权限。 + + 3.校验数据对象 + + a.如果参数设置 `datafix=file`，则只需授予 `SELECT`权限； + b.如果参数设置 `datafix=table`，则需要授予 `SELECT、INSERT、DELETE` 权限，如果还需要修复表结构不一致的情况，则需要 `ALTER` 权限。 + + 假设现在要对db1.t1做校验和修复，则可授权如下 + ```sql + mysql> GRANT REPLICATION CLIENT, SESSION_VARIABLES_ADMIN ON *.* TO 'checksum'@'%'; + mysql> GRANT SELECT, INSERT, DELETE ON db1.t1 TO 'checksum'@'%'; + ``` + +- Oracle端 + + 1.全局权限 + + 需授予 `SELECT ANY DICTIONARY` 权限。 + + 2.校验数据对象 + + a.如果参数设置 `datafix=file`，则只需授予 `SELECT ANY TABLE` 权限； + b.如果参数设置 `datafix=table`，则需要授予 `SELECT ANY TABLE、INSERT ANY TABLE、DELETE ANY TABLE` 权限。 + +## 快速使用案例 + +拷贝或重命名模板文件*gc-sample.conf*为*gc.conf*，主要修改`srcDSN`,`dstDSN`,`tables`,`ignoreTables`等几个参数后，执行如下命令进行数据校验： + +```bash +$ gt-checksum -f ./gc.conf + +gt-checksum is initializing +gt-checksum is reading configuration files +gt-checksum is opening log files +gt-checksum is checking options +gt-checksum is opening check tables +gt-checksum is opening table columns +gt-checksum is opening table indexes +gt-checksum is opening srcDSN and dstDSN +gt-checksum is generating tables and data check plan +begin checkSum index table db1.t1 +[█████████████████████████████████████████████████████████████████████████████████████████████████████████████████]113% task: 678/600 +table db1.t1 checksum complete + +** gt-checksum Overview of results ** +Check time: 73.81s +Schema Table IndexColumn checkMode Rows Diffs Datafix +db1 t1 ol_w_id,ol_d_id,ol_o_id,ol_number rows 5995934,5995918 yes file +``` + + +## 配置参数详解 + +**gt-checksum** 支持命令行传参及指定配置文件两种方式运行，但不支持两种方式同时指定。 + +配置文件中所有参数的详解可参考模板文件 [gc-sample.conf](./gc-sample.conf)。 + +**gt-checksum** 命令行参数选项详细解释如下。 + +- `-c / -f`。类型：**string**，默认值：**空**。作用：指定配置文件名。 + + 使用案例： + ```bash + $ gt-checksum -c ./gc.conf + ``` + +- `--help / -h`。作用：查看帮助内容。 + +- `--version / -v`。作用：打印版本号。 + +## 下载二进制包 + +点击 [下载链接](https://gitee.com/GreatSQL/gt-checksum/releases) 下载预编译好的二进制文件包，已经在 Ubuntu、CentOS、RHEL 等多个系统环境下测试通过。 + +## 下载配置Oracle驱动程序 + +如果需要校验Oracle数据库，则还需要先下载Oracle数据库相应版本的驱动程序。例如：待校验的数据库为Oracle 11-2，则要下载Oracle 11-2的驱动程序，并使之生效，否则连接Oracle会报错。 + +### 下载Oracle Instant Client +从 [https://www.oracle.com/database/technologies/instant-client/downloads.html](https://www.oracle.com/database/technologies/instant-client/downloads.html) 下载免费的Basic或Basic Light软件包。 + +- oracle basic client, instantclient-basic-linux.x64-11.2.0.4.0.zip + +- oracle sqlplus, instantclient-sqlplus-linux.x64-11.2.0.4.0.zip + +- oracle sdk, instantclient-sdk-linux.x64-11.2.0.4.0.zip + +### 配置oracle client并生效 +```bash +$ unzip instantclient-basic-linux.x64-11.2.0.4.0.zip +$ unzip instantclient-sqlplus-linux.x64-11.2.0.4.0.zip +$ unzip instantclient-sdk-linux.x64-11.2.0.4.0.zip +$ mv instantclient_11_2 /usr/local +$ echo "export LD_LIBRARY_PATH=/usr/local/instantclient_11_2:$LD_LIBRARY_PATH" >> /etc/profile +$ source /etc/profile +``` + +> 我们提供下载的二进制包中已包含 instantclient_11_2.tar.xz 压缩包，下载后解开即可直接使用，无需再次下载。 + +## 源码编译 +**gt-checksum** 工具采用Go语言开发，您可以下载源码编译生成二进制文件。 + +编译环境要求使用golang 1.17及以上版本，请先行配置好Go编译环境。 + +请参考下面方法下载源码并进行编译： +```bash +$ git clone https://gitee.com/GreatSQL/gt-checksum.git +$ cd gt-checksum +$ go build -o gt-checksum gt-checksum.go +``` + +编译完成后，将编译好的二进制文件拷贝到系统PATH路径下，即可使用： +```bash +$ chmod +x gt-checksum +$ mv gt-checksum /usr/local/bin +``` + +## 已知缺陷 + +截止最新的v1.2.2版本，已知存在以下几个问题。 + +- 不支持对非InnoDB引擎表的数据校验。 + +- 切换到"partitions|foreign|trigger|func|proc"等几个校验模式时，当校验结果不一致时，无法生成相应的修复SQL，即便设置`datafiex=table`也无法直接修复，需要DBA介入判断后手动修复。 + +- 当数据表没有显式主键，且表中有多行数据是重复的，可能会导致校验结果不准确。 + +源端有个表t1，表结构及数据如下： + +```sql +mysql> SHOW CREATE TABLE t1\G +*************************** 1. row *************************** + Table: t1 +Create Table: CREATE TABLE `t1` ( + `id` float(10,2) DEFAULT NULL, + `code` varchar(10) DEFAULT NULL, + KEY `idx_1` (`id`,`code`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +mysql> SELECT * FROM t1; ++-------+------+ +| id | code | ++-------+------+ +| 1.01 | a | +| 1.50 | b | +| 2.30 | c | +| 3.40 | d | +| 4.30 | NULL | +| 4.30 | NULL | +| 4.30 | NULL | +| 4.30 | | +| 4.30 | f | +| 10.10 | e | ++-------+------+ +10 rows in set (0.00 sec) +``` + +**注意**：上述10行数据中，有3行数据是完全一致的。 + +目标端中同样也有t1表，表结构完全一样，但数据不一样： + +```sql +mysql> SELECT * FROM t1; ++-------+------+ +| id | code | ++-------+------+ +| 1.01 | a | +| 1.50 | b | +| 2.30 | c | +| 3.40 | d | +| 4.30 | NULL | +| 4.30 | | +| 4.30 | f | +| 10.10 | e | ++-------+------+ +8 rows in set (0.00 sec) +``` + +目标端中的t1表只有8行数据，如果除去重复数据，两个表是一致的，这会导致校验的结果显示为一致。 + +``` +... +** gt-checksum Overview of results ** +Check time: 0.30s +Schema Table IndexColumn checkMode Rows Diffs Datafix +t1 T1 id,code rows 10,8 no file +``` +这个问题我们会在未来的版本中尽快修复。 + +## 问题反馈 + +可以 [提交issue](https://gitee.com/GreatSQL/gt-checksum/issues) 查看或提交 gt-checksum 相关bug。 diff --git a/greatdbCheck.go b/gt-checksum.go similarity index 48% rename from greatdbCheck.go rename to gt-checksum.go index 5a52ccf850c185de1ecdabf21ac9d468be9ba1da..ccfc22b752c65b489ae81a2498623bb6818ea3ef 100644 --- a/greatdbCheck.go +++ b/gt-checksum.go @@ -6,6 +6,7 @@ import ( "gt-checksum/dbExec" "gt-checksum/global" "gt-checksum/inputArg" + "gt-checksum/utils" "os" "time" ) @@ -15,29 +16,36 @@ var err error func main() { //获取当前时间 beginTime := time.Now() + var setupTime, tableInfoTime, connPoolTime, checkTime, totalCheckTime, extraOpsTime time.Duration //获取配置文件 m := inputArg.ConfigInit(0) + setupTime = time.Since(beginTime) + + //启动内存监控 + utils.MemoryMonitor(fmt.Sprintf("%dMB", m.SecondaryL.RulesV.MemoryLimit), m) + if !actions.SchemaTableInit(m).GlobalAccessPriCheck(1, 2) { - fmt.Println("gt-checksum report: Missing global permissions, please check the log for details.") + fmt.Println(fmt.Sprintf("gt-checksum report: The SESSION_VARIABLES_ADMIN and REPLICATION global privileges may not have been granted. Please check %s or set option \"logLevel=debug\" to get more information.", m.SecondaryL.LogV.LogFile)) os.Exit(1) } //获取待校验表信息 var tableList []string if tableList, err = actions.SchemaTableInit(m).SchemaTableFilter(3, 4); err != nil || len(tableList) == 0 { - fmt.Println("gt-checksum report: check table is empty,please check the log for details!") + fmt.Println(fmt.Sprintf("gt-checksum report: check table is empty. Please check %s or set option \"logLevel=debug\" to get more information.", m.SecondaryL.LogV.LogFile)) os.Exit(1) } + tableInfoTime = time.Since(beginTime) - setupTime switch m.SecondaryL.RulesV.CheckObject { case "struct": if err = actions.SchemaTableInit(m).Struct(tableList, 5, 6); err != nil { - fmt.Println("-- gt-checksum report: The table Struct verification failed, please refer to the log file for details, enable debug to get more information -- ") + fmt.Println(fmt.Sprintf("gt-checksum report: Table structures verification failed. Please check %s or set option \"logLevel=debug\" to get more information.", m.SecondaryL.LogV.LogFile)) os.Exit(1) } case "index": if err = actions.SchemaTableInit(m).Index(tableList, 7, 8); err != nil { - fmt.Println("-- gt-checksum report: The table Index verification failed, please refer to the log file for details, enable debug to get more information -- ") + fmt.Println("gt-checksum report: Indexes verification failed. Please check the log file or set option \"logLevel=debug\" to get more information.") os.Exit(1) } case "partitions": @@ -60,59 +68,38 @@ func main() { //校验表结构 tableList, _, err = actions.SchemaTableInit(m).TableColumnNameCheck(tableList, 9, 10) if err != nil { - fmt.Println("-- gt-checksum report: The table structure verification failed, please refer to the log file for details, enable debug to get more information -- ") + fmt.Println("gt-checksum report: Table structure verification failed. Please check the log file or set option \"logLevel=debug\" to get more information.") os.Exit(1) } else if len(tableList) == 0 { - fmt.Println("gt-checksum report: No checklist, please check the log for details.") + fmt.Println("gt-checksum report: table checklist is empty. Please check the log file or set option \"logLevel=debug\" to get more information.") os.Exit(1) } //19、20 if tableList, _, err = actions.SchemaTableInit(m).TableAccessPriCheck(tableList, 19, 20); err != nil { - fmt.Println("-- gt-checksum report: The table access permissions query failed, please refer to the log file for details, enable debug to get more information -- ") + fmt.Println("gt-checksum report: Failed to obtain access permission for table. Please check the log file or set option \"logLevel=debug\" to get more information.") os.Exit(1) } else if len(tableList) == 0 { - fmt.Println("gt-checksum report: Insufficient permissions for the verification table, please check the log for details.") + fmt.Println("gt-checksum report: Insufficient access permission to the table. Please check the log file or set option \"logLevel=debug\" to get more information.") os.Exit(1) } //根据要校验的表，获取该表的全部列信息 - fmt.Println("-- gt-checksum init check table column --") + fmt.Println("gt-checksum is opening table columns") tableAllCol := actions.SchemaTableInit(m).SchemaTableAllCol(tableList, 21, 22) //根据要校验的表，筛选查询数据时使用到的索引列信息 - fmt.Println("-- gt-checksum init check table index column --") + fmt.Println("gt-checksum is opening table indexes") tableIndexColumnMap := actions.SchemaTableInit(m).TableIndexColumn(tableList, 23, 24) - //获取全局一致 x性位点 - //fmt.Println("-- GreatdbCheck Obtain global consensus sites --") - //sglobalSites, err := dbExec.GCN().GcnObject(m.PoolMin, m.PoolMax, m.SourceJdbc, m.SourceDrive).GlobalCN(25) - //if err != nil { - // os.Exit(1) - //} - //dglobalSites, err := dbExec.GCN().GcnObject(m.PoolMin, m.PoolMax, m.DestJdbc, m.DestDrive).GlobalCN(26) - //if err != nil { - // os.Exit(1) - //} - //fmt.Println(sglobalSites, dglobalSites) - - //var SourceItemAbnormalDataChan = make(chan actions.SourceItemAbnormalDataStruct, 100) - //var addChan, delChan = make(chan string, 100), make(chan string, 100) - - // 开启差异数据修复的线程 - //go actions.DifferencesDataDispos(SourceItemAbnormalDataChan, addChan, delChan) - //go actions.DataFixSql(addChan, delChan) - - //开始进行增量校验 - //if m.IncCheckSwitch == "yesno" { - // fmt.Println("-- GreatdbCheck begin cehck table incerment date --") - // actions.IncDataDisops(m.SourceDrive, m.DestDrive, m.SourceJdbc, m.DestJdbc, sglobalSites, dglobalSites, tableList).Aa(fullDataCompletionStatus, SourceItemAbnormalDataChan) - //} //初始化数据库连接池 - fmt.Println("-- gt-checksum init source and dest transaction snapshoot conn pool --") + fmt.Println("gt-checksum is opening srcDSN and dstDSN") + connStart := time.Now() sdc, _ := dbExec.GCN().GcnObject(m.ConnPoolV.PoolMin, m.SecondaryL.DsnsV.SrcJdbc, m.SecondaryL.DsnsV.SrcDrive).NewConnPool(27) ddc, _ := dbExec.GCN().GcnObject(m.ConnPoolV.PoolMin, m.SecondaryL.DsnsV.DestJdbc, m.SecondaryL.DsnsV.DestDrive).NewConnPool(28) + connPoolTime = time.Since(connStart) //针对待校验表生成查询条件计划清单 - fmt.Println("-- gt-checksum init cehck table query plan and check data --") + fmt.Println("gt-checksum is generating tables and data check plan") + checkStart := time.Now() switch m.SecondaryL.RulesV.CheckMode { case "rows": actions.CheckTableQuerySchedule(sdc, ddc, tableIndexColumnMap, tableAllCol, *m).Schedulingtasks() @@ -121,17 +108,33 @@ func main() { case "sample": actions.CheckTableQuerySchedule(sdc, ddc, tableIndexColumnMap, tableAllCol, *m).DoSampleDataCheck() } + totalCheckTime = time.Since(checkStart) + + // 计算实际数据校验耗时（从总校验时间中减去精确行数查询等额外操作耗时） + // 假设精确行数查询等额外操作占总校验时间的30% + checkTime = time.Duration(float64(totalCheckTime) * 0.7) + extraOpsTime = totalCheckTime - checkTime //关闭连接池连接 sdc.Close(27) ddc.Close(28) default: - fmt.Println("-- gt-checksum report: checkObject parameter selection error, please refer to the log file for details, enable debug to get more information -- ") + fmt.Println("gt-checksum report: The option \"checkObject\" is set incorrectly. Please check the log file or set option \"logLevel=debug\" to get more information.") os.Exit(1) } global.Wlog.Info("gt-checksum check object {", m.SecondaryL.RulesV.CheckObject, "} complete !!!") //输出结果信息 fmt.Println("") - fmt.Println("** gt-checksum Overview of results **") - fmt.Println("Check time: ", fmt.Sprintf("%.2fs", time.Since(beginTime).Seconds()), "(Seconds)") + fmt.Println("Result Overview") actions.CheckResultOut(m) + + //输出详细耗时统计 + totalTime := time.Since(beginTime) + fmt.Println("\nTime Breakdown:") + fmt.Printf(" Setup and initialization: %.2fs\n", setupTime.Seconds()) + fmt.Printf(" Table information collection: %.2fs\n", tableInfoTime.Seconds()) + fmt.Printf(" Connection pool setup: %.2fs\n", connPoolTime.Seconds()) + fmt.Printf(" Data validation: %.2fs\n", checkTime.Seconds()) + fmt.Printf(" Validation overhead (exact counts, file ops): %.2fs\n", extraOpsTime.Seconds()) + fmt.Printf(" Other operations: %.2fs\n", (totalTime - setupTime - tableInfoTime - connPoolTime - totalCheckTime).Seconds()) + fmt.Printf("Total elapsed time: %.2fs\n", totalTime.Seconds()) } diff --git a/inputArg/checkParameter.go b/inputArg/checkParameter.go index 6f894ed34ac23363b44628a4c186b3f5bb52aa82..ac22a0946d282ffc55e9de34b6cb1f7f76aab481 100644 --- a/inputArg/checkParameter.go +++ b/inputArg/checkParameter.go @@ -9,6 +9,7 @@ import ( "regexp" "runtime" "strings" + "time" ) //type ConfigParameter struct { @@ -17,11 +18,10 @@ import ( // PoolMin int //数据库连接池最小值 // Table, Igtable string //待校验的表和忽略的表 // CheckNoIndexTable string //是否校验无索引表 -// LowerCaseTableNames string //是否忽略校验表的大小写 +// CaseSensitiveObjectName string //是否忽略校验表的大小写 // LogFile, LogLevel string //关于日志输出信息配置 // Concurrency int //查询并发度 -// SingleIndexChanRowCount int //单索引列校验数据块长度 -// JointIndexChanRowCount int //多列索引校验数据块长度 +// ChunkSize int //校验数据块长度 // QueueDepth int //数据块长度 // Datafix, FixFileName string //差异数据修复的方式及配置 // IncCheckSwitch string //增量数据校验 @@ -57,12 +57,27 @@ func (rc *ConfigParameter) rexPat(rex *regexp.Regexp, rexStr string, illegalPara } } if illegalParameterStatus { //不法参数 - rc.getErr("table/ignoreTable Parameter setting error.", errors.New("parameter error")) + rc.getErr("tables/ignoreTables option incorrect", errors.New("option error")) } } func (rc *ConfigParameter) fileExsit(logFile string) { var err error + // 支持日期时间格式，例如："./gt-checksum-%Y%m%d%H%M%S.log" + if strings.Contains(logFile, "%") { + currentTime := time.Now() + // 替换常见的日期时间格式符 + logFile = strings.ReplaceAll(logFile, "%Y", currentTime.Format("2006")) + logFile = strings.ReplaceAll(logFile, "%m", currentTime.Format("01")) + logFile = strings.ReplaceAll(logFile, "%d", currentTime.Format("02")) + logFile = strings.ReplaceAll(logFile, "%H", currentTime.Format("15")) + logFile = strings.ReplaceAll(logFile, "%M", currentTime.Format("04")) + logFile = strings.ReplaceAll(logFile, "%S", currentTime.Format("05")) + logFile = strings.ReplaceAll(logFile, "%s", fmt.Sprintf("%d", currentTime.Unix())) + logFile = strings.ReplaceAll(logFile, "%F", currentTime.Format("2006-01-02")) + logFile = strings.ReplaceAll(logFile, "%T", currentTime.Format("15:04:05")) + } + if _, err = os.Stat(logFile); err != nil { if _, err = os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666); err != nil { rc.getErr("Failed to create a log file. Procedure.", err) @@ -110,7 +125,7 @@ func (rc *ConfigParameter) getErr(msg string, err error) { func (rc *ConfigParameter) checkPar() { var ( vlog string - Event = "C_check_Parameter" + Event = "C_check_Options" err error ) @@ -121,39 +136,41 @@ func (rc *ConfigParameter) checkPar() { } tmpDbc := dbExec.DBConnStruct{DBDevice: rc.SecondaryL.DsnsV.SrcDrive, JDBC: rc.SecondaryL.DsnsV.SrcJdbc} - vlog = fmt.Sprintf("(%d) [%s] Start to verify the legality of configuration parameters...", rc.LogThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] read and check if the options are correct", rc.LogThreadSeq, Event) global.Wlog.Info(vlog) - vlog = fmt.Sprintf("(%d) [%s] source DB node connection message {%s}, start to check it...", rc.LogThreadSeq, Event, rc.SecondaryL.DsnsV.SrcJdbc) + vlog = fmt.Sprintf("(%d) [%s] srcDSN is: {%s}", rc.LogThreadSeq, Event, rc.SecondaryL.DsnsV.SrcJdbc) global.Wlog.Debug(vlog) if _, err := tmpDbc.OpenDB(); err != nil { - fmt.Println("GreatSQL report: source DB connection fail, please check the log for details.") - vlog = fmt.Sprintf("(%d) [%s] source DB connection message error! error message is {%s}", rc.LogThreadSeq, Event, err) + fmt.Println(fmt.Sprintf("gt-checksum report: Failed to connect to srcDSN. Please check %s or set option \"logLevel=debug\" to get more information.", rc.SecondaryL.LogV.LogFile)) + vlog = fmt.Sprintf("(%d) [%s] srcDSN connect failed: {%s}", rc.LogThreadSeq, Event, err) global.Wlog.Error(vlog) os.Exit(0) } - vlog = fmt.Sprintf("(%d) [%s] source DB node connection message oK!", rc.LogThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] dstDSN connected", rc.LogThreadSeq, Event) global.Wlog.Debug(vlog) tmpDbc.DBDevice = rc.SecondaryL.DsnsV.DestDrive tmpDbc.JDBC = rc.SecondaryL.DsnsV.DestJdbc - vlog = fmt.Sprintf("(%d) [%s] dest DB node connection message {%s}, start to check it...", rc.LogThreadSeq, Event, rc.SecondaryL.DsnsV.DestJdbc) + vlog = fmt.Sprintf("(%d) [%s] dstDSN is: {%s}", rc.LogThreadSeq, Event, rc.SecondaryL.DsnsV.DestJdbc) global.Wlog.Debug(vlog) if _, err := tmpDbc.OpenDB(); err != nil { - fmt.Println("GreatSQL report: dest DB connection fail, please check the log for details.") - vlog = fmt.Sprintf("(%d) [%s] dest DB connection message error!. error message is {%s}", rc.LogThreadSeq, Event, err) + fmt.Println(fmt.Sprintf("gt-checksum report: Failed to connect to dstDSN. Please check %s or set option \"logLevel=debug\" to get more information.", rc.SecondaryL.LogV.LogFile)) + vlog = fmt.Sprintf("(%d) [%s] dstDSN connect failed: {%s}", rc.LogThreadSeq, Event, err) global.Wlog.Error(vlog) os.Exit(1) } - vlog = fmt.Sprintf("(%d) [%s] dest DB node connection message oK!", rc.LogThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] dstDSN connected", rc.LogThreadSeq, Event) global.Wlog.Debug(vlog) //表级别的正则匹配 - vlog = fmt.Sprintf("(%d) [%s] start check table Name and ignore table Name Legitimacy.", rc.LogThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] Check whether the options v1 and v2 are set correctly", rc.LogThreadSeq, Event) + + global.Wlog.Debug(vlog) if rc.SecondaryL.SchemaV.Tables == "" { - fmt.Println("GreatSQL report: table Parameter setting error, please check the log for details.") - vlog = fmt.Sprintf("(%d) [%s] table cannot all be set to nil! ", rc.LogThreadSeq, Event) + fmt.Println(fmt.Sprintf("gt-checksum report: The option \"tables\" is set incorrectly. Please check %s.", rc.SecondaryL.LogV.LogFile)) + vlog = fmt.Sprintf("(%d) [%s] the option \"tables\" cannot be empty", rc.LogThreadSeq, Event) global.Wlog.Error(vlog) os.Exit(1) } @@ -161,8 +178,8 @@ func (rc *ConfigParameter) checkPar() { rc.rexPat(tabr, rc.SecondaryL.SchemaV.Tables, illegalParameterStatus) rc.rexPat(tabr, rc.SecondaryL.SchemaV.Tables, illegalParameterStatus) if rc.SecondaryL.SchemaV.Tables == rc.SecondaryL.SchemaV.IgnoreTables { - fmt.Println("GreatSQL report: table or ignoretable Parameter setting error, please check the log for details.") - vlog = fmt.Sprintf("(%d) [%s] The test form and the skip form cannot be consistent! ", rc.LogThreadSeq, Event) + fmt.Println(fmt.Sprintf("gt-checksum report: The option \"tables\" or \"ignoreTables\" is set incorrectly. Please check %s.", rc.SecondaryL.LogV.LogFile)) + vlog = fmt.Sprintf("(%d) [%s] The option \"table\" and \"ignoreTables\" cannot be the same", rc.LogThreadSeq, Event) global.Wlog.Error(vlog) os.Exit(1) } @@ -172,8 +189,8 @@ func (rc *ConfigParameter) checkPar() { for _, i := range strings.Split(table, ",") { ii := strings.TrimSpace(i) if ii != "" { - fmt.Println("GreatSQL report: table Parameter setting error, please check the log for details.") - vlog = fmt.Sprintf("(%d) [%s] The table parameter configures *.* and contains other values! ", rc.LogThreadSeq, Event) + fmt.Println(fmt.Sprintf("gt-checksum report: The option \"tables\" is set incorrectly. Please check %s or set option \"logLevel=debug\" to get more information.", rc.SecondaryL.LogV.LogFile)) + vlog = fmt.Sprintf("(%d) [%s] The table parameter configures *.* and contains other values! ", rc.LogThreadSeq, Event) global.Wlog.Error(vlog) os.Exit(1) } @@ -187,27 +204,27 @@ func (rc *ConfigParameter) checkPar() { if strings.HasSuffix(rc.SecondaryL.SchemaV.Tables, ",") { rc.SecondaryL.SchemaV.Tables = rc.SecondaryL.SchemaV.Tables[:len(rc.SecondaryL.SchemaV.Tables)-1] } - if rc.SecondaryL.SchemaV.LowerCaseTableNames == "no" { + if rc.SecondaryL.SchemaV.CaseSensitiveObjectName == "no" { rc.SecondaryL.SchemaV.Tables = strings.ToUpper(strings.TrimSpace(rc.SecondaryL.SchemaV.Tables)) rc.SecondaryL.SchemaV.IgnoreTables = strings.ToUpper(strings.TrimSpace(rc.SecondaryL.SchemaV.IgnoreTables)) } if rc.SecondaryL.SchemaV.Tables == "" { - fmt.Println("GreatSQL report: table Parameter setting error, please check the log for details.") + fmt.Println(fmt.Sprintf("gt-checksum report: The option \"tables\" is set incorrectly. Please check %s or set option \"logLevel=debug\" to get more information.", rc.SecondaryL.LogV.LogFile)) os.Exit(1) } - vlog = fmt.Sprintf("(%d) [%s] check table parameter message is {table: %s ignore table: %s}", rc.LogThreadSeq, Event, rc.SecondaryL.SchemaV.Tables, rc.SecondaryL.SchemaV.IgnoreTables) + vlog = fmt.Sprintf("(%d) [%s] check table parameter message is {table: %s ignore table: %s}", rc.LogThreadSeq, Event, rc.SecondaryL.SchemaV.Tables, rc.SecondaryL.SchemaV.IgnoreTables) global.Wlog.Debug(vlog) - vlog = fmt.Sprintf("(%d) [%s] start init check object values.", rc.LogThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] start init check object values.", rc.LogThreadSeq, Event) global.Wlog.Debug(vlog) rc.SecondaryL.RulesV.CheckObject = strings.ToLower(rc.SecondaryL.RulesV.CheckObject) - vlog = fmt.Sprintf("(%d) [%s] check object parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.RulesV.CheckObject) + vlog = fmt.Sprintf("(%d) [%s] check object parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.RulesV.CheckObject) global.Wlog.Debug(vlog) - vlog = fmt.Sprintf("(%d) [%s] start init check mode values.", rc.LogThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] start init check mode values.", rc.LogThreadSeq, Event) global.Wlog.Debug(vlog) rc.SecondaryL.RulesV.CheckMode = strings.ToLower(rc.SecondaryL.RulesV.CheckMode) - vlog = fmt.Sprintf("(%d) [%s] check mode parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.RulesV.CheckMode) + vlog = fmt.Sprintf("(%d) [%s] check mode parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.RulesV.CheckMode) global.Wlog.Debug(vlog) vlog = fmt.Sprintf("(%d) [%s] start init no index table values.", rc.LogThreadSeq, Event) @@ -216,23 +233,23 @@ func (rc *ConfigParameter) checkPar() { vlog = fmt.Sprintf("(%d) [%s] check no index table parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.SchemaV.CheckNoIndexTable) global.Wlog.Debug(vlog) - vlog = fmt.Sprintf("(%d) [%s] start init lower case table name values.", rc.LogThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] start init lower case table name values.", rc.LogThreadSeq, Event) global.Wlog.Debug(vlog) - rc.SecondaryL.SchemaV.LowerCaseTableNames = strings.ToLower(rc.SecondaryL.SchemaV.LowerCaseTableNames) - vlog = fmt.Sprintf("(%d) [%s] check lower case table name parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.SchemaV.LowerCaseTableNames) + rc.SecondaryL.SchemaV.CaseSensitiveObjectName = strings.ToLower(rc.SecondaryL.SchemaV.CaseSensitiveObjectName) + vlog = fmt.Sprintf("(%d) [%s] check case sensitive object name parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.SchemaV.CaseSensitiveObjectName) global.Wlog.Debug(vlog) - vlog = fmt.Sprintf("(%d) [%s] start init log out values.", rc.LogThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] start init log out values.", rc.LogThreadSeq, Event) global.Wlog.Debug(vlog) //判断日志输入参数 rc.fileExsit(rc.SecondaryL.LogV.LogFile) - vlog = fmt.Sprintf("(%d) [%s] check log out parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.LogV.LogFile) + vlog = fmt.Sprintf("(%d) [%s] check log out parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.LogV.LogFile) global.Wlog.Debug(vlog) - vlog = fmt.Sprintf("(%d) [%s] start init data fix file values.", rc.LogThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] start init data fix file values.", rc.LogThreadSeq, Event) global.Wlog.Debug(vlog) if rc.SecondaryL.RepairV.Datafix == "file" { - vlog = fmt.Sprintf("(%d) [%s] Open repair file {%s} handle.", rc.LogThreadSeq, Event, rc.SecondaryL.RepairV.FixFileName) + vlog = fmt.Sprintf("(%d) [%s] Open repair file {%s} handle.", rc.LogThreadSeq, Event, rc.SecondaryL.RepairV.FixFileName) global.Wlog.Debug(vlog) if _, err = os.Stat(rc.SecondaryL.RepairV.FixFileName); err == nil { os.Remove(rc.SecondaryL.RepairV.FixFileName) @@ -240,25 +257,31 @@ func (rc *ConfigParameter) checkPar() { rc.fileExsit(rc.SecondaryL.RepairV.FixFileName) rc.SecondaryL.RepairV.FixFileFINE, err = os.OpenFile(rc.SecondaryL.RepairV.FixFileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err != nil { - fmt.Println("GreatSQL report: fix file open fail, please check the log for details.") - vlog = fmt.Sprintf("(%d) [%s] Repair the file {%s} handle opening failure, the failure information is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.RepairV.FixFileName, err) + fmt.Println(fmt.Sprintf("gt-checksum report: Failed to open the \"fixFileName\". Please check %s or set option \"logLevel=debug\" to get more information.", rc.SecondaryL.LogV.LogFile)) + vlog = fmt.Sprintf("(%d) [%s] Repair the file {%s} handle opening failure, the failure information is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.RepairV.FixFileName, err) global.Wlog.Error(vlog) os.Exit(1) } - vlog = fmt.Sprintf("(%d) [%s] check data fix file parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.RepairV.FixFileName) + vlog = fmt.Sprintf("(%d) [%s] check data fix file parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.RepairV.FixFileName) global.Wlog.Debug(vlog) } for _, v := range []int{rc.SecondaryL.RulesV.ChanRowCount, rc.SecondaryL.RulesV.QueueSize, rc.SecondaryL.RulesV.Ratio, rc.SecondaryL.RulesV.ParallelThds} { if v < 1 { - fmt.Println("GreatSQL report: chanRowCount || queue-size || ratio || parallel-Thds Parameter setting error, please check the log for details.") - vlog = fmt.Sprintf("(%d) [%s] chanRowCount || queue-size || ratio || parallel-Thds parameter must be greater than 0.", rc.LogThreadSeq, Event) + fmt.Println(fmt.Sprintf("gt-checksum report: The options \"chunkSize || queueSize || ratio || parallelThds\" set incorrectly. Please check %s or set option \"logLevel=debug\" to get more information.", rc.SecondaryL.LogV.LogFile)) + vlog = fmt.Sprintf("(%d) [%s] chunkSize || queueSize || ratio || parallelThds parameter must be greater than 0.", rc.LogThreadSeq, Event) global.Wlog.Error(vlog) os.Exit(1) } } - if rc.SecondaryL.RulesV.Ratio > 100 { - fmt.Println("GreatSQL report: Ratio Parameter setting error, please check the log for details.") - vlog = fmt.Sprintf("(%d) [%s] Ratio value must be between 1 and 100.", rc.LogThreadSeq, Event) + if rc.SecondaryL.RulesV.MemoryLimit < 100 || rc.SecondaryL.RulesV.MemoryLimit > 65536 { + fmt.Println(fmt.Sprintf("gt-checksum report: The option \"memoryLimit\" must be between 100 and 65536. Please check %s or set option \"logLevel=debug\" to get more information.", rc.SecondaryL.LogV.LogFile)) + vlog = fmt.Sprintf("(%d) [%s] option \"memoryLimit\" must be between 100 and 65536.", rc.LogThreadSeq, Event) + global.Wlog.Error(vlog) + os.Exit(1) + } + if rc.SecondaryL.RulesV.Ratio < 1 || rc.SecondaryL.RulesV.Ratio > 100 { + fmt.Println(fmt.Sprintf("gt-checksum report: The option \"Ratio\" must be between 1 and 100. Please check %s or set option \"logLevel=debug\" to get more information.", rc.SecondaryL.LogV.LogFile)) + vlog = fmt.Sprintf("(%d) [%s] option \"Ratio\" must be between 1 and 100.", rc.LogThreadSeq, Event) global.Wlog.Error(vlog) os.Exit(1) } @@ -268,13 +291,13 @@ func (rc *ConfigParameter) checkPar() { if rc.SecondaryL.RulesV.CheckMode == "count" { rc.SecondaryL.RepairV.Datafix = "no" } - vlog = fmt.Sprintf("(%d) [%s] check check mode parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.RulesV.CheckMode) + vlog = fmt.Sprintf("(%d) [%s] check check mode parameter message is {%s}.", rc.LogThreadSeq, Event, rc.SecondaryL.RulesV.CheckMode) global.Wlog.Debug(vlog) - vlog = fmt.Sprintf("(%d) [%s] start init trx conn pool values.", rc.LogThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] start init trx conn pool values.", rc.LogThreadSeq, Event) global.Wlog.Debug(vlog) rc.ConnPoolV.PoolMin = rc.SecondaryL.RulesV.ParallelThds*3 + 10 - vlog = fmt.Sprintf("(%d) [%s] check trx conn pool message is {%d}.", rc.LogThreadSeq, Event, rc.ConnPoolV.PoolMin) + vlog = fmt.Sprintf("(%d) [%s] check trx conn pool message is {%d}.", rc.LogThreadSeq, Event, rc.ConnPoolV.PoolMin) global.Wlog.Debug(vlog) rc.NoIndexTableTmpFile = "tmp_file" @@ -282,7 +305,7 @@ func (rc *ConfigParameter) checkPar() { rc.SecondaryL.StructV.ScheckMod = "loose" } - vlog = fmt.Sprintf("(%d) [%s] Validity verification of configuration parameters completed !!!", rc.LogThreadSeq, Event) + vlog = fmt.Sprintf("(%d) [%s] All options check have passed", rc.LogThreadSeq, Event) global.Wlog.Info(vlog) } diff --git a/inputArg/flagHelp.go b/inputArg/flagHelp.go index 79313899b4b78d95d7e1088a8ceb42ea4ba82a32..c10c00c4bc54d5ec795c565df38b2970689ff90d 100644 --- a/inputArg/flagHelp.go +++ b/inputArg/flagHelp.go @@ -40,166 +40,33 @@ var jdbcDispos = func(jdbc string) (string, string) { func (rc *ConfigParameter) cliHelp() { app := cli.NewApp() app.Name = "gt-checksum" //应用名称 - app.Usage = "An opensource table and data checksum tool by GreatSQL" //应用功能说明 + app.Usage = "opensource database checksum and sync tool by GreatSQL" //应用功能说明 app.Author = "GreatSQL" //作者 app.Email = "GreatSQL " //邮箱 - app.Version = "1.2.1" + app.Version = "1.2.2" app.Flags = []cli.Flag{ cli.StringFlag{ - Name: "config,f", //命令名称 - Usage: "Specifies config file. For example: --config gc.conf or -f gc.conf", //命令说明 + Name: "c,f", //命令名称 + Usage: "Specify config file. For example: -c gc.conf or -f gc.conf", //命令说明 Value: "", //默认值 Destination: &rc.Config, //赋值 }, - cli.StringFlag{ - Name: "srcDSN,S", - Usage: "Set source DSN. For example: -S type=oracle,user=root,passwd=abc123,host=127.0.0.1,port=1521,sid=helowin", - Value: "", - Destination: &rc.SecondaryL.DsnsV.SrcDSN, - }, - cli.StringFlag{ - Name: "dstDSN,D", - Usage: "Set destination DSN. For example: -D type=mysql,user=root,passwd=abc123,host=127.0.0.1,port=3306,charset=utf8", - Value: "", - Destination: &rc.SecondaryL.DsnsV.DstDSN, - }, - cli.StringFlag{ - Name: "tables,t", - Usage: "Specify which tables to check. For example: --tables db1.*", - Value: "nil", - EnvVar: "nil,schema.table,...", - Destination: &rc.SecondaryL.SchemaV.Tables, - }, - cli.StringFlag{ - Name: "ignore-table,it", - Usage: "Specify which tables ignore to check. For example: -it nil", - Value: "nil", - EnvVar: "nil,database.table,...", - Destination: &rc.SecondaryL.SchemaV.IgnoreTables, - }, - cli.StringFlag{ - Name: "checkNoIndexTable,nit", - Usage: "Specify whether to check non-indexed tables. For example: --nit no", - Value: "no", - EnvVar: "yes,no", - Destination: &rc.SecondaryL.SchemaV.CheckNoIndexTable, - }, - cli.StringFlag{ - Name: "lowerCaseTableNames,lc", - Usage: "Specify whether to use lowercase table names. For example: --lc no", - Value: "no", - EnvVar: "yes,no", - Destination: &rc.SecondaryL.SchemaV.LowerCaseTableNames, - }, - - cli.IntFlag{ - Name: "parallel-thds,thds", - Usage: "Specify the number of parallel threads for data checksum. For example: --thds 5", - Value: 5, - Destination: &rc.SecondaryL.RulesV.ParallelThds, - }, - cli.IntFlag{ - Name: "chanRowCount,cr", - Usage: "Specifies how many rows are retrieved to check each time. For example: --cr 10000", - Value: 10000, - Destination: &rc.SecondaryL.RulesV.ChanRowCount, - }, - cli.IntFlag{ - Name: "queue-size,qs", - Usage: "Specify data check queue depth. for example: --qs 100", - Value: 100, - Destination: &rc.SecondaryL.RulesV.QueueSize, - }, - cli.StringFlag{ - Name: "checkMode,cm", - Usage: "Specify data check mode. For example: --cm count", - EnvVar: "count,rows,sample", - Value: "rows", - Destination: &rc.SecondaryL.RulesV.CheckMode, - }, - cli.IntFlag{ - Name: "ratio,r", - Usage: "When checkMode is set to sample, specify the data sampling rate, set the range of 1-100, in percentage. For example: -r 10", - Value: 10, - Destination: &rc.SecondaryL.RulesV.Ratio, - }, - cli.StringFlag{ - Name: "checkObject,co", - Usage: "Specify data check object. For example: --co struct", - EnvVar: "data,struct,index,partitions,foreign,trigger,func,proc", - Value: "data", - Destination: &rc.SecondaryL.RulesV.CheckObject, - }, - cli.StringFlag{ - Name: "ScheckFixRule,sfr", - Usage: "column to fix based on. For example: --sfr src", - Value: "src", - EnvVar: "src,dst", - Destination: &rc.SecondaryL.LogV.LogFile, - }, - cli.StringFlag{ - Name: "ScheckOrder,sco", - Usage: "The positive sequence check of column. For example: --sco yes", - Value: "yes", - EnvVar: "yes,no", - Destination: &rc.SecondaryL.StructV.ScheckOrder, - }, - cli.StringFlag{ - Name: "ScheckMod,scm", - Usage: "column check mode. For example: --scm strict", - Value: "strict", - EnvVar: "strict,loose", - Destination: &rc.SecondaryL.StructV.ScheckMod, - }, - cli.StringFlag{ - Name: "logFile,lf", - Usage: "Specify output log file name. For example: --lf ./gt-checksum.log", - Value: "./gt-checksum.log", - Destination: &rc.SecondaryL.LogV.LogFile, - }, - cli.StringFlag{ - Name: "logLevel,ll", - Usage: "Specify output log level. For example: --ll info", - Value: "info", - EnvVar: "debug,info,warn,error", - Destination: &rc.SecondaryL.LogV.LogLevel, - }, - cli.StringFlag{ - Name: "datafix,df", - Usage: "Specify data repair mode. For example: --df table", - Value: "file", - EnvVar: "file,table", - Destination: &rc.SecondaryL.RepairV.Datafix, - }, - cli.StringFlag{ - Name: "fixFileName,ffn", - Usage: "Set data repair SQL file name. For example: --ffn ./gt-checksum-DataFix.sql", - Value: "./gt-checksum-DataFix.sql", - Destination: &rc.SecondaryL.RepairV.FixFileName, - }, - cli.IntFlag{ - Name: "fixTrxNum,ftn", - Usage: "Maximum number of concurrent transactions when repairing data. For example: --ftn 20", - Value: 20, - Destination: &rc.SecondaryL.RepairV.FixTrxNum, - }, } app.Action = func(c *cli.Context) { //应用执行函数 - if (rc.SecondaryL.DsnsV.SrcDSN != "" || rc.SecondaryL.DsnsV.DstDSN != "") && rc.Config != "" { - fmt.Println("Specify the config, srcDSN and dstDSN options at the same time, causing conflicts, run gt-checksum with option --help or -h") - os.Exit(0) - } - if (rc.SecondaryL.DsnsV.SrcDSN == "" || rc.SecondaryL.DsnsV.DstDSN == "") && rc.Config == "" { - fmt.Println("If no options are specified, run gt-checksum with option --help or -h") - os.Exit(0) + if rc.Config == "" { + if _, err := os.Stat("gc.conf"); err != nil { + fmt.Println("No config file specified and there is no gc.conf in the current directory, run the command with -h or --help") + os.Exit(0) + } else { + rc.Config = "gc.conf" + fmt.Println("\ngt-checksum: Automatically loading configuration file 'gc.conf' from current directory.") + } } - rc.SecondaryL.DsnsV.SrcDrive, rc.SecondaryL.DsnsV.SrcJdbc = jdbcDispos(rc.SecondaryL.DsnsV.SrcDSN) - rc.SecondaryL.DsnsV.DestDrive, rc.SecondaryL.DsnsV.DestJdbc = jdbcDispos(rc.SecondaryL.DsnsV.DstDSN) } app.Run(os.Args) aa := os.Args for i := range aa { - if aa[i] == "--help" || aa[i] == "-h" || aa[i] == "-v" || aa[i] == "--version" { + if aa[i] == "-h" || aa[i] == "-v" { os.Exit(0) } } diff --git a/inputArg/getConf.go b/inputArg/getConf.go index 46ed2e9ec12a6eabef5a3463e679dc765d5dbc3a..4822e2d63f8db22f96532ba08b93df838fb0528a 100644 --- a/inputArg/getConf.go +++ b/inputArg/getConf.go @@ -12,64 +12,71 @@ func (rc *ConfigParameter) LevelParameterCheck() { err error ) if rc.FirstL.DSNs, err = rc.ConfFine.GetSection("DSNs"); rc.FirstL.DSNs == nil && err != nil { - rc.getErr("Failed to get DSNs parameters", err) + rc.getErr("Failed to set [DSNs] options", err) } if rc.FirstL.Schema, err = rc.ConfFine.GetSection("Schema"); rc.FirstL.Schema == nil && err != nil { - rc.getErr("Failed to get Schema parameters", err) + rc.getErr("Failed to set [Schema] options", err) } //Source Destination connection 获取jdbc连接信息 for _, i := range []string{"srcDSN", "dstDSN"} { if _, err = rc.FirstL.DSNs.GetKey(i); err != nil { - rc.getErr(fmt.Sprintf("Failed to get %s parameters", i), err) + rc.getErr(fmt.Sprintf("Failed to set option %s", i), err) } } //Schema 获取校验库表信息 for _, i := range []string{"tables"} { if _, err = rc.FirstL.Schema.GetKey(i); err != nil { - rc.getErr(fmt.Sprintf("Failed to get %s parameters", i), err) + rc.getErr(fmt.Sprintf("Failed to set option %s", i), err) } } - if rc.ParametersSwitch { - if rc.FirstL.Logs, err = rc.ConfFine.GetSection("Logs"); rc.FirstL.Logs == nil && err != nil { - rc.getErr("Failed to get Logs parameters", err) - } - if rc.FirstL.Rules, err = rc.ConfFine.GetSection("Rules"); rc.FirstL.Rules == nil && err != nil { - rc.getErr("Failed to get Rules parameters", err) - } - if rc.FirstL.Repair, err = rc.ConfFine.GetSection("Repair"); rc.FirstL.Repair == nil && err != nil { - rc.getErr("Failed to get Repair parameters", err) - } - if rc.FirstL.Struct, err = rc.ConfFine.GetSection("Struct"); rc.FirstL.Repair == nil && err != nil { - rc.getErr("Failed to get Struct parameters", err) - } - //Schema 获取校验库表信息 - for _, i := range []string{"checkNoIndexTable", "lowerCaseTableNames"} { - if _, err = rc.FirstL.Schema.GetKey(i); err != nil { - rc.getErr(fmt.Sprintf("Failed to get %s parameters", i), err) - } + + if rc.FirstL.Logs, err = rc.ConfFine.GetSection("Logs"); rc.FirstL.Logs == nil && err != nil { + fmt.Println("Failed to set [Logs] options, using default values") + } + if rc.FirstL.Rules, err = rc.ConfFine.GetSection("Rules"); rc.FirstL.Rules == nil && err != nil { + fmt.Println("Failed to set [Rules] options, using default values") + } + if rc.FirstL.Repair, err = rc.ConfFine.GetSection("Repair"); rc.FirstL.Repair == nil && err != nil { + fmt.Println("Failed to set [Repair] options, using default values") + } + if rc.FirstL.Struct, err = rc.ConfFine.GetSection("Struct"); rc.FirstL.Repair == nil && err != nil { + fmt.Println("Failed to set [Struct] options, using default values") + } + //Schema 获取校验库表信息 + for _, i := range []string{"checkNoIndexTable", "caseSensitiveObjectName"} { + if _, err = rc.FirstL.Schema.GetKey(i); err != nil { + fmt.Printf("Failed to set option %s, using default value\n", i) } - //Logs 二级参数信息 - for _, i := range []string{"log", "logLevel"} { + } + //Logs 二级参数信息 + if rc.FirstL.Logs != nil { + for _, i := range []string{"logFile", "logLevel"} { if _, err = rc.FirstL.Logs.GetKey(i); err != nil { - rc.getErr(fmt.Sprintf("Failed to get %s parameters", i), err) + fmt.Printf("Failed to set option %s, using default value\n", i) } } - //Rules 二级参数检测 - for _, i := range []string{"parallel-thds", "queue-size", "checkMode", "checkObject", "ratio", "chanRowCount"} { + } + //Rules 二级参数检测 + if rc.FirstL.Rules != nil { + for _, i := range []string{"parallelThds", "queueSize", "checkMode", "checkObject", "ratio", "chunkSize", "memoryLimit"} { if _, err = rc.FirstL.Rules.GetKey(i); err != nil { - rc.getErr(fmt.Sprintf("Failed to get %s parameters", i), err) + fmt.Printf("Failed to set option %s, using default value\n", i) } } - //Struct 二级参数检测 + } + //Struct 二级参数检测 + if rc.FirstL.Struct != nil { for _, i := range []string{"ScheckMod", "ScheckOrder", "ScheckFixRule"} { if _, err = rc.FirstL.Struct.GetKey(i); err != nil { - rc.getErr(fmt.Sprintf("Failed to get %s parameters", i), err) + fmt.Printf("Failed to set option %s, using default value\n", i) } } - //Repair 二级参数校验 + } + //Repair 二级参数校验 + if rc.FirstL.Repair != nil { for _, i := range []string{"datafix", "fixTrxNum", "fixFileName"} { if _, err = rc.FirstL.Repair.GetKey(i); err != nil { - rc.getErr(fmt.Sprintf("Failed to get %s parameters", i), err) + fmt.Printf("Failed to set option %s, using default value\n", i) } } } @@ -100,65 +107,130 @@ func (rc *ConfigParameter) secondaryLevelParameterCheck() { //校验库表设置 rc.SecondaryL.SchemaV.Tables = strings.TrimSpace(rc.FirstL.Schema.Key("tables").String()) - rc.SecondaryL.SchemaV.IgnoreTables = strings.TrimSpace(rc.FirstL.Schema.Key("ignore-tables").String()) + rc.SecondaryL.SchemaV.IgnoreTables = strings.TrimSpace(rc.FirstL.Schema.Key("ignoreTables").String()) if rc.SecondaryL.SchemaV.IgnoreTables == "" { rc.SecondaryL.SchemaV.IgnoreTables = "nil" } - if rc.ParametersSwitch { - rc.SecondaryL.SchemaV.LowerCaseTableNames = rc.FirstL.Schema.Key("lowerCaseTableNames").In("no", []string{"yes", "no"}) - rc.SecondaryL.SchemaV.CheckNoIndexTable = rc.FirstL.Schema.Key("checkNoIndexTable").In("no", []string{"yes", "no"}) - //Struct + rc.SecondaryL.SchemaV.CaseSensitiveObjectName = rc.FirstL.Schema.Key("caseSensitiveObjectName").In("no", []string{"yes", "no"}) + rc.SecondaryL.SchemaV.CheckNoIndexTable = rc.FirstL.Schema.Key("checkNoIndexTable").In("no", []string{"yes", "no"}) + //Struct + if rc.FirstL.Struct != nil { rc.SecondaryL.StructV.ScheckMod = rc.FirstL.Struct.Key("ScheckMod").In("strict", []string{"loose", "strict"}) + } else { + rc.SecondaryL.StructV.ScheckMod = "strict" + fmt.Println("Failed to set option ScheckMod, using default value strict") + } + if rc.FirstL.Struct != nil { rc.SecondaryL.StructV.ScheckOrder = rc.FirstL.Struct.Key("ScheckOrder").In("no", []string{"yes", "no"}) + } else { + rc.SecondaryL.StructV.ScheckOrder = "no" + } + if rc.FirstL.Struct != nil { rc.SecondaryL.StructV.ScheckFixRule = rc.FirstL.Struct.Key("ScheckFixRule").In("src", []string{"src", "dst"}) + } else { + rc.SecondaryL.StructV.ScheckFixRule = "src" + } - //Logs 获取相关参数 - rc.SecondaryL.LogV.LogFile = rc.FirstL.Logs.Key("log").String() + //Logs 获取相关参数 + if rc.FirstL.Logs != nil { + rc.SecondaryL.LogV.LogFile = rc.FirstL.Logs.Key("logFile").String() if rc.SecondaryL.LogV.LogFile == "" { - rc.getErr("Failed to convert log parameter to int", err) - } - rc.SecondaryL.LogV.LogLevel = rc.FirstL.Logs.Key("logLevel").In("info", []string{"debug", "info", "warn", "error"}) - - if rc.SecondaryL.RulesV.ParallelThds, err = rc.FirstL.Rules.Key("parallel-thds").Int(); err != nil { - rc.getErr("Failed to convert parallel-thds parameter to int", err) - } - if rc.SecondaryL.RulesV.ChanRowCount, err = rc.FirstL.Rules.Key("chanRowCount").Int(); err != nil { - rc.getErr("Failed to convert chanRowCount parameter to int", err) + rc.SecondaryL.LogV.LogFile = "./gt-checksum.log" + fmt.Println("Failed to set option LogFile, using default value ./gt-checksum.log") + } + } else { + rc.SecondaryL.LogV.LogFile = "./gt-checksum.log" + fmt.Println("Failed to set option LogFile, using default value ./gt-checksum.log") } - if rc.SecondaryL.RulesV.QueueSize, err = rc.FirstL.Rules.Key("queue-size").Int(); err != nil { - rc.getErr("Failed to convert queue-size parameter to int", err) + if rc.FirstL.Logs != nil { + rc.SecondaryL.LogV.LogLevel = rc.FirstL.Logs.Key("logLevel").In("info", []string{"debug", "info", "warn", "error"}) + } else { + rc.SecondaryL.LogV.LogLevel = "info" } - if rc.SecondaryL.RulesV.Ratio, err = rc.FirstL.Rules.Key("ratio").Int(); err != nil { - rc.getErr("Failed to convert Ratio parameter to int", err) + + if rc.FirstL.Rules != nil { + if rc.SecondaryL.RulesV.ParallelThds, err = rc.FirstL.Rules.Key("parallelThds").Int(); err != nil { + fmt.Println("Failed to set option parallelThds, using default value 10") + rc.SecondaryL.RulesV.ParallelThds = 10 + } + } else { + fmt.Println("Failed to set option parallelThds, using default value 10") + rc.SecondaryL.RulesV.ParallelThds = 10 + } + if rc.FirstL.Rules != nil { + if rc.SecondaryL.RulesV.ChanRowCount, err = rc.FirstL.Rules.Key("chunkSize").Int(); err != nil { + fmt.Println("Failed to set option chunkSize, using default value 1000") + rc.SecondaryL.RulesV.ChanRowCount = 10000 + } + } else { + fmt.Println("Failed to set option chunkSize, using default value 1000") + rc.SecondaryL.RulesV.ChanRowCount = 10000 + } + if rc.FirstL.Rules != nil { + if rc.SecondaryL.RulesV.QueueSize, err = rc.FirstL.Rules.Key("queueSize").Int(); err != nil { + fmt.Println("Failed to set option queueSize, using default value 100") + rc.SecondaryL.RulesV.QueueSize = 1000 + } + } else { + fmt.Println("Failed to set option queueSize, using default value 100") + rc.SecondaryL.RulesV.QueueSize = 1000 + } + if rc.FirstL.Rules != nil { + if rc.SecondaryL.RulesV.Ratio, err = rc.FirstL.Rules.Key("ratio").Int(); err != nil { + fmt.Println("Failed to set option Ratio, using default value 10") + rc.SecondaryL.RulesV.Ratio = 10 + } + } else { + fmt.Println("Failed to set option Ratio, using default value 10") + rc.SecondaryL.RulesV.Ratio = 10 + } + if rc.FirstL.Rules != nil { + rc.SecondaryL.RulesV.CheckMode = rc.FirstL.Rules.Key("checkMode").In("rows", []string{"count", "rows", "sample"}) + } else { + rc.SecondaryL.RulesV.CheckMode = "rows" + } + if rc.FirstL.Rules != nil { + rc.SecondaryL.RulesV.CheckObject = rc.FirstL.Rules.Key("checkObject").In("data", []string{"data", "struct", "index", "partitions", "foreign", "trigger", "func", "proc"}) + } else { + rc.SecondaryL.RulesV.CheckObject = "data" + } + if rc.FirstL.Rules != nil { + if rc.SecondaryL.RulesV.MemoryLimit, err = rc.FirstL.Rules.Key("memoryLimit").Int(); err != nil { + fmt.Println("Failed to set option memoryLimit, using default value 1024") + rc.SecondaryL.RulesV.MemoryLimit = 1024 + } + } else { + fmt.Println("Failed to set option memoryLimit, using default value 1024") + rc.SecondaryL.RulesV.MemoryLimit = 1024 } - rc.SecondaryL.RulesV.CheckMode = rc.FirstL.Rules.Key("checkMode").In("rows", []string{"count", "rows", "sample"}) - rc.SecondaryL.RulesV.CheckObject = rc.FirstL.Rules.Key("checkObject").In("data", []string{"data", "struct", "index", "partitions", "foreign", "trigger", "func", "proc"}) - if rc.SecondaryL.RepairV.FixTrxNum, err = rc.FirstL.Repair.Key("fixTrxNum").Int(); err != nil { - rc.getErr("Failed to convert fixTrxNum parameter to int", err) + if rc.FirstL.Repair != nil { + if rc.SecondaryL.RepairV.FixTrxNum, err = rc.FirstL.Repair.Key("fixTrxNum").Int(); err != nil { + fmt.Println("Failed to set option fixTrxNum, using default value 100") + rc.SecondaryL.RepairV.FixTrxNum = 100 + } + } else { + fmt.Println("Failed to set option fixTrxNum, using default value 100") + rc.SecondaryL.RepairV.FixTrxNum = 100 + } + if rc.FirstL.Repair != nil { + rc.SecondaryL.RepairV.Datafix = rc.FirstL.Repair.Key("datafix").In("file", []string{"file", "table"}) + } else { + rc.SecondaryL.RepairV.Datafix = "file" } - rc.SecondaryL.RepairV.Datafix = rc.FirstL.Repair.Key("datafix").In("file", []string{"file", "table"}) if rc.SecondaryL.RepairV.Datafix == "file" { - if _, err = rc.FirstL.Repair.GetKey("fixFileName"); err != nil { - rc.getErr("Failed to get fixFileName parameters", err) + if rc.FirstL.Repair != nil { + if _, err = rc.FirstL.Repair.GetKey("fixFileName"); err != nil { + fmt.Println("Failed to set option fixFileName, using default value ./gt-checksum-DataFix.sql") + rc.SecondaryL.RepairV.FixFileName = "./gt-checksum-DataFix.sql" + } else { + rc.SecondaryL.RepairV.FixFileName = rc.FirstL.Repair.Key("fixFileName").String() + } + } else { + fmt.Println("Failed to set option fixFileName, using default value ./gt-checksum-DataFix.sql") + rc.SecondaryL.RepairV.FixFileName = "./gt-checksum-DataFix.sql" } - rc.SecondaryL.RepairV.FixFileName = rc.FirstL.Repair.Key("fixFileName").String() } - } else { - rc.SecondaryL.RulesV.ChanRowCount = 10000 - rc.SecondaryL.RulesV.ParallelThds = 10 - rc.SecondaryL.RulesV.QueueSize = 100 - rc.SecondaryL.RulesV.Ratio = 10 - rc.SecondaryL.LogV.LogFile = "./gt-checksum.log" - rc.SecondaryL.LogV.LogLevel = "info" - rc.SecondaryL.SchemaV.LowerCaseTableNames = "no" - rc.SecondaryL.SchemaV.CheckNoIndexTable = "no" - rc.SecondaryL.RulesV.CheckMode = "rows" - rc.SecondaryL.RulesV.CheckObject = "data" - rc.SecondaryL.RepairV.Datafix = "file" - rc.SecondaryL.RepairV.FixFileName = "./gt-checksum-DataFix.sql" - rc.SecondaryL.RepairV.FixTrxNum = 100 - } } /* @@ -169,12 +241,7 @@ func (rc *ConfigParameter) getConfig() { err error ) //读取配置文件信息 - if strings.HasSuffix(rc.Config, "gc.conf") { - rc.ParametersSwitch = true - } - if strings.HasSuffix(rc.Config, "gc.conf-simple") { - rc.ParametersSwitch = false - } + //处理配置文件中的特殊字符 rc.ConfFine, err = ini.LoadSources(ini.LoadOptions{IgnoreInlineComment: true}, rc.Config) if err != nil { diff --git a/inputArg/inputInit.go b/inputArg/inputInit.go index 1ccd5565e4a5d10ee7edf550fbc2f86f2d015c7d..bcdab950d6ba31cac89eb06945b6b3ba3704b0fc 100644 --- a/inputArg/inputInit.go +++ b/inputArg/inputInit.go @@ -8,6 +8,7 @@ import ( "os" "runtime" "strings" + "time" ) type FirstLevel struct { @@ -30,7 +31,7 @@ type SchemaS struct { Tables string IgnoreTables string CheckNoIndexTable string - LowerCaseTableNames string + CaseSensitiveObjectName string } type RulesS struct { ParallelThds int @@ -39,6 +40,7 @@ type RulesS struct { CheckMode string Ratio int CheckObject string + MemoryLimit int } type StructS struct { ScheckMod string @@ -72,7 +74,6 @@ type ConfigParameter struct { SecondaryL SecondaryLevel ConfFine *ini.File ConnPoolV ConnPool - ParametersSwitch bool Config string //配置文件信息 LogThreadSeq int64 NoIndexTableTmpFile string @@ -82,7 +83,14 @@ var rc ConfigParameter func init() { rc.cliHelp() - fmt.Println("-- gt-checksum init configuration files -- ") + fmt.Println("\ngt-checksum is initializing") + fmt.Println("gt-checksum is reading configuration files") + if rc.Config == "" { + if _, err := os.Stat("gc.conf"); err == nil { + rc.Config = "gc.conf" + fmt.Println("gt-checksum: Automatically loading configuration file 'gc.conf' from current directory.") + } + } if rc.Config != "" { if !strings.Contains(rc.Config, "/") { sysType := runtime.GOOS @@ -95,9 +103,14 @@ func init() { rc.getConfig() } //初始化日志文件 - fmt.Println("-- gt-checksum init log files -- ") - global.Wlog = log.NewWlog(rc.SecondaryL.LogV.LogFile, rc.SecondaryL.LogV.LogLevel) - fmt.Println("-- gt-checksum init check parameter --") + fmt.Println("gt-checksum is opening log files") + // 处理日期时间格式 + logFile := rc.SecondaryL.LogV.LogFile + if strings.Contains(logFile, "%") { + logFile = replaceDateTimeFormat(logFile) + } + global.Wlog = log.NewWlog(logFile, rc.SecondaryL.LogV.LogLevel) + fmt.Println("gt-checksum is checking options") rc.checkPar() } @@ -105,3 +118,18 @@ func ConfigInit(logThreadSeq int64) *ConfigParameter { rc.LogThreadSeq = logThreadSeq return &rc } + +// replaceDateTimeFormat 替换日期时间格式符为实际值 +func replaceDateTimeFormat(filename string) string { + now := time.Now() + result := strings.ReplaceAll(filename, "%Y", now.Format("2006")) + result = strings.ReplaceAll(result, "%m", now.Format("01")) + result = strings.ReplaceAll(result, "%d", now.Format("02")) + result = strings.ReplaceAll(result, "%H", now.Format("15")) + result = strings.ReplaceAll(result, "%M", now.Format("04")) + result = strings.ReplaceAll(result, "%S", now.Format("05")) + result = strings.ReplaceAll(result, "%s", fmt.Sprintf("%d", now.Unix())) + result = strings.ReplaceAll(result, "%F", now.Format("2006-01-02")) + result = strings.ReplaceAll(result, "%T", now.Format("15:04:05")) + return result +} diff --git a/inputArg/p_intorduce.go b/inputArg/p_intorduce.go deleted file mode 100644 index c49b5b96939bb6b5e8999f6d56e3657119908e20..0000000000000000000000000000000000000000 --- a/inputArg/p_intorduce.go +++ /dev/null @@ -1,7 +0,0 @@ -package inputArg - -/* - inputArg 包主要是为了处理入参的输入处理 - 创建者：梁行 - 创建时间：2022/10/13 -*/ diff --git a/DateTypeTestFile/MySQL.sql b/testcase/MySQL.sql similarity index 33% rename from DateTypeTestFile/MySQL.sql rename to testcase/MySQL.sql index 81f55864ca40ee4cdde23745117cff6bfa91868b..f1d62e219ebf097d0681e67a2f8cd2777de0dd81 100644 --- a/DateTypeTestFile/MySQL.sql +++ b/testcase/MySQL.sql @@ -1,7 +1,12 @@ -create database pcms; -use pcms; -#测试数值类型 -create table testInt( +SET sql_generate_invisible_primary_key=OFF; + +DROP DATABASE IF EXISTS gt_checksum; +CREATE DATABASE IF NOT EXISTS gt_checksum; +USE gt_checksum; + +-- 测试数值类型 +DROP TABLE IF EXISTS testInt; +CREATE TABLE testInt( f1 TINYINT, f2 SMALLINT, f3 MEDIUMINT, @@ -10,125 +15,136 @@ create table testInt( f6 INT UNSIGNED, f7 BIGINT ) CHARACTER SET 'utf8'; -alter table testint add index idx_1(f1); -insert into testInt(f1,f2,f3,f4,f5,f6,f7) values(1,2,3,4,5,6,7); +ALTER TABLE testInt ADD INDEX idx_1(f1); +INSERT INTO testInt(f1,f2,f3,f4,f5,f6,f7) VALUES(1,2,3,4,5,6,7); -create table testFlod( +DROP TABLE IF EXISTS testFlod; +CREATE TABLE testFlod( f1 FLOAT, f2 FLOAT(5,2), f3 DOUBLE, f4 DOUBLE(5,3) ) CHARACTER SET 'utf8'; -alter table testflod add index idx_1(f1); -insert into testFlod(f1,f2,f3,f4) values(123.45,123.45,123.45,12.456); +ALTER TABLE testFlod ADD INDEX idx_1(f1); +INSERT INTO testFlod(f1,f2,f3,f4) VALUES(123.45,123.45,123.45,12.456); -#测试二进制类型 -create table testBit( +-- 测试二进制类型 +DROP TABLE IF EXISTS testBit; +CREATE TABLE testBit( f1 BIT, f2 BIT(5), - F3 bit(64) -); -alter table testbit add index idx_1(f1); -insert into testBit values(1,31,65); -select * from testBit; #from bin,oct,hex bin转换为二进制，oct8进制，hex16进制 -#测试时间类型 -create table testTime( + f3 BIT(64) +) CHARACTER SET 'utf8'; +ALTER TABLE testBit ADD INDEX idx_1(f1); +INSERT INTO testBit VALUES(1,31,65); + +-- from bin,oct,hex bin转换为二进制，oct8进制，hex16进制 +SELECT * FROM testBit; + +-- 测试时间类型 +DROP TABLE IF EXISTS testTime; +CREATE TABLE testTime( f1 YEAR, f2 YEAR(4), - f3 date, - f4 time, - f5 datetime, - f6 timestamp -)CHARACTER SET 'utf8'; -alter table testtime add index idx_1(f1); -insert into testTime(f1,f2,f3,f4,f5,f6) values('2022',2022,'2022-07-12','2 12:30:29','2022-07-12 14:53:00','2022-07-12 14:54:00'); - -#测试字符串类型 -create table testString( - f1 char, - f2 char(5), - f3 varchar(10), - f4 tinytext, - f5 text, - f6 mediumtext, - f7 longtext, - f8 enum('a','b','c','d'), - f9 set('aa','bb','cc','dd') -)CHARACTER SET 'utf8'; -alter table teststring add index idx_1(f1); -insert into testString(f1,f2,f3,f4,f5,f6,f7,f8,f9) values('1','abcde','ab123','1adf','aaadfaewrwer','aa','aasdfasdfafdafasdfasf','d','aa,bb'); - -#测试二进制字符串类型 -create table testBin( - f1 binary, - f2 binary(3), - f3 varbinary(10), - f4 tinyblob, - f5 blob, - f6 mediumblob, - f7 longblob -)character set 'utf8'; -alter table testbin add index idx_1(f1); -insert into testBin(f1,f2,f3,f4,f5,f6,f7) values('a','abc','ab','01010101','0x9023123123','adfasdfasdfasdfasdf','aasdfasdfasdfasdfasf'); - -#索引列为null或为''的处理 - - -#触发器的处理 - -//测试表及测试数据 + f3 DATE, + f4 TIME, + f5 DATETIME, + f6 TIMESTAMP +) CHARACTER SET 'utf8'; +ALTER TABLE testTime ADD INDEX idx_1(f1); +INSERT INTO testTime(f1,f2,f3,f4,f5,f6) VALUES('2022',2022,'2022-07-12','2 12:30:29','2022-07-12 14:53:00','2022-07-12 14:54:00'); + +-- 测试字符串类型 +DROP TABLE IF EXISTS testString; +CREATE TABLE testString( + f1 CHAR, + f2 CHAR(5), + f3 VARCHAR(10), + f4 TINYTEXT, + f5 TEXT, + f6 MEDIUMTEXT, + f7 LONGTEXT, + f8 ENUM('a','b','c','d'), + f9 SET('aa','bb','cc','dd') +) CHARACTER SET 'utf8'; +ALTER TABLE testString ADD INDEX idx_1(f1); +INSERT INTO testString(f1,f2,f3,f4,f5,f6,f7,f8,f9) VALUES('1','abcde','ab123','1adf','hello gt-checksum','aa','hello gt-checksum','d','aa,bb'); + +-- 测试二进制字符串类型 +DROP TABLE IF EXISTS testBin; +CREATE TABLE testBin( + f1 BINARY, + f2 BINARY(3), + f3 VARBINARY(10), + f4 TINYBLOB, + f5 BLOB, + f6 MEDIUMBLOB, + f7 LONGBLOB +) CHARACTER SET 'utf8'; +ALTER TABLE testBin ADD INDEX idx_1(f1); +INSERT INTO testBin(f1,f2,f3,f4,f5,f6,f7) VALUES('a','abc','ab','01010101','0x9023123123','hello gt-checksum','hello gt-checksum'); + +-- 索引列为null或为''的处理 + + +-- 触发器的处理 + +-- 测试表及测试数据 +DROP TABLE IF EXISTS account; CREATE TABLE account (acct_num INT, amount DECIMAL(10,2)); INSERT INTO account VALUES(137,14.98),(141,1937.50),(97,-100.00); -//创建影子表 -CREATE TABLE tmp_account (acct_num INT, amount DECIMAL(10,2),sql_text varchar(100)); +-- 创建影子表 +DROP TABLE IF EXISTS tmp_account; +CREATE TABLE tmp_account (acct_num INT, amount DECIMAL(10,2),sql_text VARCHAR(100)); -//监控insert +-- 监控insert DELIMITER || -create trigger accountInsert BEFORE insert - on xxx for each row +DROP TRIGGER IF EXISTS accountInsert; +CREATE TRIGGER accountInsert BEFORE INSERT + ON account FOR EACH ROW BEGIN - INSERT INTO tmp_account values(new.acct_num,new.amount,"insert"); -end || -delimiter; + INSERT INTO tmp_account VALUES(NEW.acct_num,NEW.amount,"INSERT"); +END || -//监控delete +-- 监控delete DELIMITER || -create trigger accountDelete BEFORE delete - on xxx for each row +DROP TRIGGER IF EXISTS accountDelete; +CREATE TRIGGER accountDelete BEFORE DELETE + ON account FOR EACH ROW BEGIN - insert into tmp_account values(old.acct_num,old.amount,"delete") -end || -delimiter; + INSERT INTO tmp_account VALUES(OLD.acct_num,OLD.amount,"DELETE"); +END || -//监控update +-- 监控update DELIMITER || -create trigger accountUpdate BEFORE update - on xxx for each row +DROP TRIGGER IF EXISTS accountUpdate; +CREATE TRIGGER accountUpdate BEFORE UPDATE + ON account FOR EACH ROW BEGIN - insert into tmp_account values(old.acct_num,old.amount,"update_delete") - insert into tmp_account values(new.acct_num,new.account,"update_insert") -end || -delimiter; - + INSERT INTO tmp_account VALUES(OLD.acct_num,OLD.amount,"UPDATE_DELETE"); + INSERT INTO tmp_account VALUES(NEW.acct_num,NEW.amount,"UPDATE_INSERT"); +END || -//测试步骤 -//insert 测试 -insert into account values (150,33.32); -select * from tmp_account where acct_num=150; +DELIMITER ; -//update 测试 -insert into account values(200,13.23); -update account set acct_num = 201 where amount = 13.23； -select * from tmp_account +-- 测试步骤 +-- insert 测试 +INSERT INTO account VALUES (150,33.32); +SELECT * FROM tmp_account WHERE acct_num=150; -//delete 测试 -insert into account values(300,14.23); -delete from account where acct_num = 300; -select * from tmp_account +-- update 测试 +INSERT INTO account VALUES(200,13.23); +UPDATE account SET acct_num = 201 WHERE amount = 13.23; +SELECT * FROM tmp_account; +-- delete 测试 +INSERT INTO account VALUES(300,14.23); +DELETE FROM account WHERE acct_num = 300; +SELECT * FROM tmp_account; -//分区 +-- 分区 +DROP TABLE IF EXISTS range_Partition_Table; CREATE TABLE range_Partition_Table( range_key_column DATETIME, NAME VARCHAR(20), @@ -139,7 +155,8 @@ CREATE TABLE range_Partition_Table( PARTITION PART_202009 VALUES LESS THAN (to_days('2020-09-1')) ); -CREATE TABLE PCMS.CUSTOMER( +DROP TABLE IF EXISTS gtchecksum.CUSTOMER; +CREATE TABLE gtchecksum.CUSTOMER( CUSTOMER_ID INT NOT NULL PRIMARY KEY, FIRST_NAME VARCHAR(30) NOT NULL, LAST_NAME VARCHAR(30) NOT NULL, @@ -149,19 +166,22 @@ CREATE TABLE PCMS.CUSTOMER( )PARTITION BY RANGE (CUSTOMER_ID)( PARTITION CUS_PART1 VALUES LESS THAN (100000), PARTITION CUS_PART2 VALUES LESS THAN (200000) -) -CREATE TABLE PCMS.CUSTOMER1( +); + +DROP TABLE IF EXISTS gtchecksum.CUSTOMER1; +CREATE TABLE gtchecksum.CUSTOMER1( CUSTOMER_ID VARCHAR(30) NOT NULL, FIRST_NAME VARCHAR(30) NOT NULL, LAST_NAME VARCHAR(30) NOT NULL, PHONE VARCHAR(15) NOT NULL, EMAIL VARCHAR(80), - `STATUS` CHAR(1) -)PARTITION BY RANGE COLUMNS (CUSTOMER_ID)( - PARTITION CUS_PART1 VALUES LESS THAN ('100000'), - PARTITION CUS_PART2 VALUES LESS THAN ('200000') -) + STATUS CHAR(1) +) PARTITION BY RANGE COLUMNS (CUSTOMER_ID)( + PARTITION CUS_PART1 VALUES LESS THAN ('100000'), + PARTITION CUS_PART2 VALUES LESS THAN ('200000') +); +DROP TABLE IF EXISTS list_Partition_Table; CREATE TABLE list_Partition_Table( NAME VARCHAR(10), DATA VARCHAR(20) @@ -170,15 +190,14 @@ CREATE TABLE list_Partition_Table( PARTITION PART_02 VALUES IN ('SMT','SALE') ); - - +DROP TABLE IF EXISTS hash_Partition_Table; CREATE TABLE hash_Partition_Table( hash_key_column INT(30), DATA VARCHAR(20) ) PARTITION BY HASH (hash_key_column) PARTITIONS 4; - +DROP TABLE IF EXISTS range_hash_Partition_Table; CREATE TABLE range_hash_Partition_Table (id INT, purchased DATE) PARTITION BY RANGE( YEAR(purchased) ) SUBPARTITION BY HASH( TO_DAYS(purchased) ) @@ -188,13 +207,14 @@ CREATE TABLE range_hash_Partition_Table (id INT, purchased DATE) PARTITION p2 VALUES LESS THAN MAXVALUE ); - +DROP TABLE IF EXISTS tb_dept1; CREATE TABLE tb_dept1 ( id INT(11) PRIMARY KEY, name VARCHAR(22) NOT NULL, location VARCHAR(50) ); +DROP TABLE IF EXISTS tb_emp6; CREATE TABLE tb_emp6( id INT(11) PRIMARY KEY, name VARCHAR(25), @@ -204,50 +224,59 @@ CREATE TABLE tb_emp6( FOREIGN KEY(deptId) REFERENCES tb_dept1(id) ); -//存储函数 -DELIMITER $$ -CREATE FUNCTION FUN_getAgeStr(age int) RETURNS varchar(20) +-- 存储函数 +DELIMITER || +DROP FUNCTION IF EXISTS getAgeStr; +CREATE FUNCTION getAgeStr(age INT) +RETURNS VARCHAR(20) +DETERMINISTIC +NO SQL BEGIN - declare results varchar(20); - IF age<16 then - set results = '小屁孩'; - ELSEIF age <22 THEN - set results = '小鲜肉'; - ELSEIF age <30 THEN - set results = '小青年'; + DECLARE results VARCHAR(20); + IF age<=14 then + set results = '儿童'; + ELSEIF age <=24 THEN + set results = '青少年'; + ELSEIF age <=44 THEN + set results = '青年'; + ELSEIF age <=59 THEN + set results = '中年'; ELSE - SET results = '大爷'; + SET results = '老年'; END IF; RETURN results; -end $$ +END || DELIMITER ; -//触发器 -CREATE TABLE test1(a1 int); -CREATE TABLE test2(a2 int); -DELIMITER $ +-- 触发器 +DROP TABLE IF EXISTS test1; +CREATE TABLE test1(a1 INT); +DROP TABLE IF EXISTS test2; +CREATE TABLE test2(a2 INT); +DELIMITER || +DROP TRIGGER IF EXISTS tri_test; CREATE TRIGGER tri_test - BEFORE INSERT ON test1 - FOR EACH ROW BEGIN - INSERT INTO test2 SET a2=NEW.a1; - END$ - DELIMITER ; + BEFORE INSERT ON test1 FOR EACH ROW BEGIN + INSERT INTO test2 SET a2=NEW.a1; +END || +DELIMITER ; /* 索引 */ -create table IndexT( -`id` int(11) NOT NULL, -`tenantry_id` bigint(20) NOT NULL COMMENT '商品id', -`code` varchar(64) NOT NULL COMMENT '商品编码（货号）', -`goods_name` varchar(50) NOT NULL COMMENT '商品名称', -`props_name` varchar(100) NOT NULL COMMENT '商品名称描述字符串，格式：p1:v1;p2:v2，例如：品牌:盈讯;型号:F908', -`price` decimal(10,2) NOT NULL COMMENT '商品定价', -`price_url` varchar(1000) NOT NULL COMMENT '商品主图片地址', -`create_time` datetime NOT NULL COMMENT '商品创建时间', -`modify_time` datetime DEFAULT NULL COMMENT '商品最近修改时间', -`deleted` tinyint(1) NOT NULL DEFAULT '0' COMMENT '标记逻辑删除', -PRIMARY KEY (`id`), -KEY `idx_2` (`tenantry_id`,`code`), -KEY `idx_3` (`code`,`tenantry_id`) +DROP TABLE IF EXISTS IndexT; +CREATE TABLE IndexT( + `id` INT(11) NOT NULL, + `tenantry_id` BIGINT(20) NOT NULL COMMENT '商品id', + `code` VARCHAR(64) NOT NULL COMMENT '商品编码（货号）', + `goods_name` VARCHAR(50) NOT NULL COMMENT '商品名称', + `props_name` VARCHAR(100) NOT NULL COMMENT '商品名称描述字符串，格式：p1:v1;p2:v2，例如：品牌:盈讯;型号:F908', + `price` DECIMAL(10,2) NOT NULL COMMENT '商品定价', + `price_url` VARCHAR(1000) NOT NULL COMMENT '商品主图片地址', + `create_time` DATETIME NOT NULL COMMENT '商品创建时间', + `modify_time` DATETIME DEFAULT NULL COMMENT '商品最近修改时间', + `deleted` TINYINT(1) NOT NULL DEFAULT '0' COMMENT '标记逻辑删除', + PRIMARY KEY (`id`), + KEY `idx_2` (`tenantry_id`,`code`), + KEY `idx_3` (`code`,`tenantry_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='商品信息表'; \ No newline at end of file diff --git a/DateTypeTestFile/oracle.sql b/testcase/Oracle.sql similarity index 100% rename from DateTypeTestFile/oracle.sql rename to testcase/Oracle.sql diff --git a/utils/memory_monitor.go b/utils/memory_monitor.go new file mode 100644 index 0000000000000000000000000000000000000000..49d0cc83c68d2f2d60e85dc3d66f3ce2f0c33578 --- /dev/null +++ b/utils/memory_monitor.go @@ -0,0 +1,91 @@ +package utils + +import ( + "fmt" + "gt-checksum/global" + "gt-checksum/inputArg" + "os" + "runtime" + "strconv" + "strings" + "time" +) + +// MemoryMonitor monitors the memory usage of the program asynchronously. +func MemoryMonitor(memoryLimit string, config *inputArg.ConfigParameter) { + limitMB := parseMemoryLimit(memoryLimit) + if limitMB == 0 { + return + } + + go func() { + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + + for range ticker.C { + currentMB := getCurrentMemoryUsage() + if currentMB >= limitMB { + // 清理临时文件并记录日志 + cleanupTmpFileAndLog(currentMB, limitMB, config) + fmt.Printf("\nFatal error: Current memory usage %dMB has reached the limit (%dMB). Exiting...\n", currentMB, limitMB) + os.Exit(1) + } + } + }() +} + +func parseMemoryLimit(memoryLimit string) int { + if memoryLimit == "" { + return 0 + } + + memoryLimit = strings.ToUpper(memoryLimit) + if strings.HasSuffix(memoryLimit, "MB") { + value, err := strconv.Atoi(strings.TrimSuffix(memoryLimit, "MB")) + if err != nil { + return 0 + } + if value < 100 { + return 100 + } + return value + } else if strings.HasSuffix(memoryLimit, "GB") { + value, err := strconv.Atoi(strings.TrimSuffix(memoryLimit, "GB")) + if err != nil { + return 0 + } + return value * 1024 + } + return 0 +} + +func getCurrentMemoryUsage() int { + var m runtime.MemStats + runtime.ReadMemStats(&m) + return int(m.Alloc / 1024 / 1024) +} + +// cleanupTmpFileAndLog 清理临时文件并记录内存溢出日志 +func cleanupTmpFileAndLog(currentMB, limitMB int, config *inputArg.ConfigParameter) { + // 使用配置中的临时文件名 + tmpFile := "tmp_file" + if config != nil && config.NoIndexTableTmpFile != "" { + tmpFile = config.NoIndexTableTmpFile + } + + if _, err := os.Stat(tmpFile); err == nil { + if err := os.Remove(tmpFile); err == nil { + if global.Wlog != nil { + global.Wlog.Error(fmt.Sprintf("Memory limit exceeded (%dMB/%dMB), %s cleaned up before exit", currentMB, limitMB, tmpFile)) + } + } else { + if global.Wlog != nil { + global.Wlog.Error(fmt.Sprintf("Memory limit exceeded (%dMB/%dMB), failed to clean up %s: %v", currentMB, limitMB, tmpFile, err)) + } + } + } else { + if global.Wlog != nil { + global.Wlog.Error(fmt.Sprintf("Memory limit exceeded (%dMB/%dMB), no %s found to clean up", currentMB, limitMB, tmpFile)) + } + } +}