Go与Python平铺嵌套JSON及其速度比较

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

{
    "first": "Dale",
    "last": "Murphy",
    "age": 44,
    "nets": ["ig", "fb", "value1", "value2"],
    "submap": {
        "subfirst": "Dale",
        "sublast": ["Murphy", "value3", "value4"],
        "subsub":{"subsub":["value5","value6","value7","value8","value9"]}
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

[
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"Murphy","subsub":"value5"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"Murphy","subsub":"value5"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"Murphy","subsub":"value5"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"Murphy","subsub":"value5"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"value3","subsub":"value5"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"value3","subsub":"value5"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"value3","subsub":"value5"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"value3","subsub":"value5"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"value4","subsub":"value5"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"value4","subsub":"value5"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"value4","subsub":"value5"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"value4","subsub":"value5"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"Murphy","subsub":"value6"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"Murphy","subsub":"value6"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"Murphy","subsub":"value6"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"Murphy","subsub":"value6"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"value3","subsub":"value6"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"value3","subsub":"value6"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"value3","subsub":"value6"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"value3","subsub":"value6"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"value4","subsub":"value6"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"value4","subsub":"value6"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"value4","subsub":"value6"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"value4","subsub":"value6"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"Murphy","subsub":"value7"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"Murphy","subsub":"value7"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"Murphy","subsub":"value7"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"Murphy","subsub":"value7"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"value3","subsub":"value7"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"value3","subsub":"value7"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"value3","subsub":"value7"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"value3","subsub":"value7"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"value4","subsub":"value7"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"value4","subsub":"value7"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"value4","subsub":"value7"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"value4","subsub":"value7"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"Murphy","subsub":"value8"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"Murphy","subsub":"value8"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"Murphy","subsub":"value8"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"Murphy","subsub":"value8"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"value3","subsub":"value8"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"value3","subsub":"value8"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"value3","subsub":"value8"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"value3","subsub":"value8"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"value4","subsub":"value8"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"value4","subsub":"value8"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"value4","subsub":"value8"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"value4","subsub":"value8"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"Murphy","subsub":"value9"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"Murphy","subsub":"value9"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"Murphy","subsub":"value9"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"Murphy","subsub":"value9"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"value3","subsub":"value9"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"value3","subsub":"value9"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"value3","subsub":"value9"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"value3","subsub":"value9"},
{"first":"Dale","last":"Murphy","age":"44","nets":"ig","subfirst":"Dale","sublast":"value4","subsub":"value9"},
{"first":"Dale","last":"Murphy","age":"44","nets":"fb","subfirst":"Dale","sublast":"value4","subsub":"value9"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value1","subfirst":"Dale","sublast":"value4","subsub":"value9"},
{"first":"Dale","last":"Murphy","age":"44","nets":"value2","subfirst":"Dale","sublast":"value4","subsub":"value9"}
]

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100

package main

import (
	"encoding/json"
	"fmt"
	"time"
)

func flattenJson(b []byte) ([]map[string]interface{}, error) {
	var jsonObj map[string]interface{}
	err := json.Unmarshal(b, &jsonObj)
	if err != nil {
		return nil, err
	}

	return flatten(jsonObj, make(map[string]interface{})), nil
}

func flatten(jsonObj map[string]interface{}, result map[string]interface{}) []map[string]interface{} {
	tempResults := []map[string]interface{}{make(map[string]interface{})}
	for k, v := range jsonObj {
		switch value := v.(type) {
		case map[string]interface{}:
			nestedResults := flatten(value, result)
			newTempResults := []map[string]interface{}{}
			for _, nested := range nestedResults {
				for _, item := range tempResults {
					newItem := make(map[string]interface{})
					for key, val := range item {
						newItem[key] = val
					}
					for key, val := range nested {
						newItem[key] = val
					}
					newTempResults = append(newTempResults, newItem)
				}
			}
			tempResults = newTempResults
		case []interface{}:
			newTempResults := []map[string]interface{}{}
			for _, elem := range value {
				for _, item := range tempResults {
					newItem := make(map[string]interface{})
					for key, val := range item {
						newItem[key] = val
					}
					newItem[k] = elem
					newTempResults = append(newTempResults, newItem)
				}
			}
			tempResults = newTempResults
		default:
			for _, item := range tempResults {
				item[k] = value
			}
		}
	}

	// Merge each item with result
	finalResults := []map[string]interface{}{}
	for _, item := range tempResults {
		newItem := make(map[string]interface{})
		for key, val := range result {
			newItem[key] = val
		}
		for key, val := range item {
			newItem[key] = val
		}
		finalResults = append(finalResults, newItem)
	}

	return finalResults
}

func main() {
	start := time.Now()
	jsonData := []byte(`{
		"first": "Dale",
		"last": "Murphy",
		"age": 44,
		"nets": ["ig", "fb", "value1", "value2"],
		"submap": {
			"subfirst": "Dale",
			"sublast": ["Murphy", "value3", "value4"],
            "subsub":{"subsub":["value5","value6","value7","value8","value9"]}
		}
}`)
	for i := 0; i < 100000; i++ {
		flattenJson(jsonData)
	}
	fmt.Println(time.Since(start))
}
// (base) C:\Users\Administrator\Desktop\hmd>go run stsd.go 
// 4.287609s
// (base) C:\Users\Administrator\Desktop\hmd>go run stsd.go
// 4.513107s
// (base) C:\Users\Administrator\Desktop\hmd>go run stsd.go
// 4.1299503s
// (base) C:\Users\Administrator\Desktop\hmd>go version
// go version go1.21.4 windows/amd64

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

import time

def flattenjson(json_obj, normalresult=None):
    if normalresult is None:
        normalresult = {}
    # 用于存储处理过程中的临时结果
    temp_results = [{}]
    for k, v in json_obj.items():
        if isinstance(v, dict):
            # 递归处理嵌套字典，并合并结果
            nested_results = flattenjson(v)
            temp_results = [dict(item, **nested) for nested in nested_results for item in temp_results]
        elif isinstance(v, list):
            # 处理列表，为每个元素创建新的字典
            temp_results = [dict(item, **{k: j}) for j in v for item in temp_results]
        else:
            # 处理普通键值对，加入到临时结果中
            for item in temp_results:
                item[k] = v
    # 将普通结果合并到每个字典中
    final_results = [dict(item, **normalresult) for item in temp_results]
    return final_results

if __name__ == '__main__':
    start = time.time()
    jsonstr = {
            "first": "Dale",
            "last": "Murphy",
            "age": 44,
            "nets": ["ig", "fb", "value1", "value2"],
            "submap": {
                "subfirst": "Dale",
                "sublast": ["Murphy", "value3", "value4"],
                "subsub":{"subsub":["value5","value6","value7","value8","value9"]}
            }
    }
    for _ in range(100000):
        flattenjson(jsonstr)
    print(time.time()-start)
# (base) C:\Users\Administrator\Desktop\hmd>python test.py
# 3.0182690620422363
# (base) C:\Users\Administrator\Desktop\hmd>python test.py
# 2.991424560546875
# (base) C:\Users\Administrator\Desktop\hmd>python test.py
# 3.039586067199707
# (base) C:\Users\Administrator\Desktop\hmd>python
# Python 3.10.9 | packaged by Anaconda, Inc. | (main, Mar  1 2023, 18:18:15) [MSC v.1916 64 bit (AMD64)] on win32
# Type "help", "copyright", "credits" or "license" for more information.
# >>>