Golang内存错误-将JSON写入文件的程序

我正在编写一个程序,该程序生成并将JSON数据写入文件。但是此解决方案不可扩展。 如果我尝试在JSON中生成超过500万条记录,则会引发内存错误。

这是代码,

package main

import (
    "github.com/icrowley/fake"
    "github.com/chilts/sid"
    "encoding/json"
    "fmt"
    //"io/ioutil"
    "os"
    //"testing"
    "time"
)

func makeData(n int) interface{} {
    datas := []map[string]interface{}{}
    for i := 0; i < n; i++ {
        data := map[string]interface{}{
            "dn" : "cn=user",
            "employeeNumber" : i,
            "givenName" : fake.FirstName(),
            "sn" : fake.LastName(),
            "fullName" : fake.FullName(),
            "mail" : fake.EmailAddress(),
            "objectClass"  :[]string{"a","b"},
            "TelephoneNumber" : fake.Phone(),
            "homephone" : fake.Phone(),
            "Street" :fake.Street(),
            "PostalAddress" : fake.StreetAddress(),
            "cn" : "user",
            "doc_id" : sid.Id(),
            "role" : fake.JobTitle(),
            "language" : fake.Language(),
            "cdsAdmAccountPwd" : fake.Word(),
            "pager" : fake.Phone(),
            "description" : fake.Words()  ,
        }
        datas = append(datas, data)
    }

    return datas
}

func makeTimestamp() int64 {
    return time.Now().UnixNano() / int64(time.Millisecond)
}

func main(){
start := makeTimestamp()
file, _ := os.OpenFile("big_encode.json", os.O_CREATE, os.ModePerm)
defer file.Close()  
encoder := json.NewEncoder(file) 
encoder.Encode(makeData(10))
end := makeTimestamp()
fmt.Println(end-start)
}

如您所见,数据将保存所有数据,如果RAM已满,则将引发内存错误。如何使此代码可伸缩。

输出文件中的内容,

[{"PostalAddress":"Maryland Circle 77","Street":"Barnett Lane","TelephoneNumber":"476-27-48","cdsAdmAccountPwd":"veritatis","cn":"user","description":"sunt suscipit dicta sunt sit","dn":"cn=user","doc_id":"1589187077699147000-4249579699234406614","employeeNumber":0,"fullName":"Joshua Patterson","givenName":"Debra","homephone":"1-904-520-51-57","language":"Gujarati","mail":"PhillipRobertson@DabZ.com","objectClass":["a","b"],"pager":"583-41-02","role":"Financial Analyst","sn":"Miller"},{"PostalAddress":"Sachtjen Road 49","Street":"Monica Court","TelephoneNumber":"1-415-297-82-75","cdsAdmAccountPwd":"explicabo","cn":"user","description":"aut cumque ut a dolorem","dn":"cn=user","doc_id":"1589187077699147000-4249579699234406615","employeeNumber":1,"fullName":"Rachel Cooper","givenName":"Melissa","homephone":"8-289-853-70-93","language":"Māori","mail":"DonaldArnold@Gabcube.info","objectClass":["a","b"],"pager":"5-577-904-31-87","role":"Health Coach I","sn":"Day"},{"PostalAddress":"Brickson Park Point 14","Street":"Sugar Road","TelephoneNumber":"7-042-335-66-37","cdsAdmAccountPwd":"qui","cn":"user","description":"magnam laborum delectus","dn":"cn=user","doc_id":"1589187077699147000-4249579699234406616","employeeNumber":2,"fullName":"Jennifer Murphy","givenName":"Clarence","homephone":"7-840-073-06-27","language":"Belarusian","mail":"AlbertCarpenter@Meembee.gov","objectClass":["a","b"],"pager":"0-485-991-52-43","role":"Computer Systems Analyst III","sn":"Cunningham"},{"PostalAddress":"Cardinal Avenue 18","Street":"Gateway Alley","TelephoneNumber":"7-522-918-64-14","cdsAdmAccountPwd":"iste","cn":"user","description":"qui","dn":"cn=user","doc_id":"1589187077699147000-4249579699234406617","employeeNumber":3,"fullName":"Sara Torres","givenName":"Jose","homephone":"4-683-962-84-63","language":"Tamil","mail":"3Hicks@Twitterworks.com","objectClass":["a","b"],"pager":"0-838-285-79-52","role":"Design Engineer","sn":"Greene"},{"PostalAddress":"Hazelcrest Road 92","Street":"Lakewood Gardens Terrace","TelephoneNumber":"8-677-576-84-41","cdsAdmAccountPwd":"commodi","cn":"user","description":"natus et","dn":"cn=user","doc_id":"1589187077699147000-4249579699234406618","employeeNumber":4,"fullName":"Harry Vasquez","givenName":"Kathy","homephone":"000-49-32","language":"Portuguese","mail":"libero@Janyx.gov","objectClass":["a","b"],"pager":"661-46-12","role":"Paralegal","sn":"Franklin"},{"PostalAddress":"Prairie Rose Court 40","Street":"Arkansas Crossing","TelephoneNumber":"975-44-67","cdsAdmAccountPwd":"quam","cn":"user","description":"in eos ad accusantium","dn":"cn=user","doc_id":"1589187077699147000-4249579699234406619","employeeNumber":5,"fullName":"Clarence Larson","givenName":"John","homephone":"887-58-07","language":"Ndebele","mail":"aut_doloribus_est@Voomm.gov","objectClass":["a","b"],"pager":"400-72-17","role":"Accounting Assistant III","sn":"Thomas"},{"PostalAddress":"Myrtle Hill 77","Street":"Thompson Circle","TelephoneNumber":"796-96-80","cdsAdmAccountPwd":"cumque","cn":"user","description":"consequatur cum","dn":"cn=user","doc_id":"1589187077699147000-4249579699234406620","employeeNumber":6,"fullName":"Emily Medina","givenName":"David","homephone":"2-255-950-00-51","language":"New Zealand Sign Language","mail":"PeterCrawford@Livefish.biz","objectClass":["a","b"],"pager":"6-513-996-29-46","role":"Speech Pathologist","sn":"Castillo"},{"PostalAddress":"Debra Alley 14","Street":"Commercial Avenue","TelephoneNumber":"559-50-87","cdsAdmAccountPwd":"eius","cn":"user","description":"alias cupiditate","dn":"cn=user","doc_id":"1589187077699147000-4249579699234406621","employeeNumber":7,"fullName":"Pamela Taylor","givenName":"Carolyn","homephone":"926-20-15","language":"Fijian","mail":"hic_molestiae@Divanoodle.net","objectClass":["a","b"],"pager":"9-632-826-96-58","role":"Developer II","sn":"Green"},{"PostalAddress":"Debra Road 73","Street":"Gale Avenue","TelephoneNumber":"4-775-370-55-45","cdsAdmAccountPwd":"fuga","cn":"user","description":"qui sapiente consectetur officiis ipsum","dn":"cn=user","doc_id":"1589187077699147000-4249579699234406622","employeeNumber":8,"fullName":"Benjamin Davis","givenName":"Rachel","homephone":"9-958-885-52-84","language":"Amharic","mail":"qEllis@Wordware.net","objectClass":["a","b"],"pager":"8-040-233-94-81","role":"Software Test Engineer II","sn":"Sanchez"},{"PostalAddress":"Sloan Point 97","Street":"Graedel Point","TelephoneNumber":"5-562-273-19-05","cdsAdmAccountPwd":"voluptatem","cn":"user","description":"corporis alias","dn":"cn=user","doc_id":"1589187077699147000-4249579699234406623","employeeNumber":9,"fullName":"Kenneth Baker","givenName":"Jerry","homephone":"9-815-502-34-27","language":"Portuguese","mail":"4West@Skiba.name","objectClass":["a","b"],"pager":"5-392-810-32-61","role":"Nuclear Power Engineer","sn":"West"}]

它是json对象的数组。

评论