From 1994b1583807609beb1cc4f3b0d4fec28fa34469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20H=C3=B6rnquist?= <hugo@lysator.liu.se> Date: Tue, 3 Nov 2020 12:44:34 +0100 Subject: [PATCH] Start adding pagination again. --- fetch/fetch.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/fetch/fetch.go b/fetch/fetch.go index c671c6a..2d363aa 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -3,6 +3,7 @@ package fetch import ( "fmt" "log" + "math" "crypto/tls" "encoding/json" @@ -51,7 +52,7 @@ func Init(pw string) { var m2 PuppetclassMessage - fff("puppetclasses", &m2) + fff_pages("puppetclasses", &m2) for group, entries := range m2.Results { log.Print("Handling ", group) @@ -90,11 +91,19 @@ func fff (url string, message interface{}) { func fff_pages(url string, message IsMessage) { fff(url, &message) log.Printf("message = %+v", message.GetMessage()) - // TODO - // logic here for fetching additional pages - // (preferably async) - // IsMessage should require a Merge method - // TODO also figure out pointer stuff + + m := message.GetMessage() + pagecount := int(math.Ceil(m.Total / m.Per_page)) + for i := 2; i < pagecount; i++ { + // TODO this kind of works for dictionaries, since json.Unmarshal + // reuses an existing dictionary. I believe that it however + // "fails" if the same key is present in multiple responses, and + // that it doesn't work for list responses. + // TODO also, all these should run in paralell + fff(url + fmt.Sprintf("?page=%d", i), + &message) + log.Printf("message = %+v", message.GetMessage()) + } } func GetClassList(host HostRecord) []string { -- GitLab