From f477cc7fdd9d5c725bde88fb87dedcf5a53517fb Mon Sep 17 00:00:00 2001 From: michaellindman Date: Thu, 4 Jul 2019 05:27:41 +0100 Subject: [PATCH] async get requests --- api.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- data.go | 67 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 151 insertions(+), 6 deletions(-) diff --git a/api.go b/api.go index 2614c9c..a652570 100755 --- a/api.go +++ b/api.go @@ -2,6 +2,7 @@ package request import ( "encoding/json" + "fmt" "html" "io/ioutil" "net/http" @@ -16,12 +17,47 @@ func url(path string) string { return string("https://forum.0cd.xyz/" + path) } +// HTTPResp Async HTTP Request +type HTTPResp struct { + ID int + Resp []byte + Err logger.HTTPError +} + +// AsyncGet sends Async GET requests +func AsyncGet(urls map[int]string) ([]*HTTPResp, *logger.HTTPError) { + ch := make(chan *HTTPResp) + response := []*HTTPResp{} + + for id, url := range urls { + go func(i int, u string) { + resp, err := Request("t/" + u) + err = logger.HTTPErr(http.StatusOK) + ch <- &HTTPResp{i, resp, *err} + }(id, url) + } +loop: + for { + select { + case r := <-ch: + response = append(response, r) + if len(response) == len(urls) { + break loop + } + case <-time.After(120 * time.Millisecond): + fmt.Printf(".") + //return nil, logger.HTTPErr(http.StatusRequestTimeout) + } + } + return response, nil +} + // Request sends GET to path func Request(path string) ([]byte, *logger.HTTPError) { req, err := http.NewRequest("GET", url(path), nil) if err != nil { logger.ErrorLog("Error reading request. ", err) - return nil, &logger.HTTPError{Status: "500 Internal Server Error", StatusCode: http.StatusInternalServerError} + return nil, logger.HTTPErr(http.StatusInternalServerError) } head, er := Header() @@ -36,12 +72,14 @@ func Request(path string) ([]byte, *logger.HTTPError) { resp, err := client.Do(req) if err != nil { logger.ErrorLog("Error reading request. ", err) - return nil, &logger.HTTPError{Status: "500 Internal Server Error", StatusCode: http.StatusInternalServerError} + fmt.Println(err) + return nil, logger.HTTPErr(http.StatusInternalServerError) } body, err := ioutil.ReadAll(resp.Body) if err != nil { logger.ErrorLog("Error reading request. ", err) - return nil, &logger.HTTPError{Status: "500 Internal Server Error", StatusCode: http.StatusInternalServerError} + fmt.Println(err) + return nil, logger.HTTPErr(http.StatusInternalServerError) } defer resp.Body.Close() @@ -49,7 +87,7 @@ func Request(path string) ([]byte, *logger.HTTPError) { return body, nil } logger.GetLog("GET %d %s\n", resp.StatusCode, html.EscapeString(url(path))) - return nil, &logger.HTTPError{Status: resp.Status, StatusCode: resp.StatusCode} + return nil, logger.HTTPErr(resp.StatusCode) } // Category returns category json data @@ -101,6 +139,50 @@ func GetTopics(path string) (topics TagTopics, err *logger.HTTPError) { return } +// GetTopics2 gets topic list from tag +func GetTopics2(path string) (m map[int]string) { + resp, err := Request("tags/" + path) + if err != nil { + return + } + var topics TagTopics + json.Unmarshal(resp, &topics) + m = make(map[int]string) + for _, topics := range topics.TopicList.Topics { + m[topics.ID] = topics.Slug + } + return +} + +// AsyncTopics request for topics using async +func AsyncTopics(path string) map[string]interface{} { + var topic TopicsList + resp, err := AsyncGet(GetTopics2(path)) + if err != nil { + m := structs.Map(err) + return m + } + for _, topics := range resp { + var t Topic + json.Unmarshal(topics.Resp, &t) + for i := 0; i < len(t.PostStream.Posts); i++ { + if t.PostStream.Posts[i].PostNumber != 1 { + t.PostStream.Posts[i].Cooked = "" + } + } + t.Details.CreatedBy.AvatarTemplate = strings.ReplaceAll(t.Details.CreatedBy.AvatarTemplate, "{size}", "120") + s := strings.SplitAfter(t.PostStream.Posts[0].Cooked, "

") + t.PostStream.Posts[0].Cooked = s[0] + r := strings.ReplaceAll(t.PostStream.Posts[0].Cooked, "href=\"/u/", "href=\""+url("")+"u/") + t.PostStream.Posts[0].Cooked = r + ts, _ := time.Parse("2006-01-02T15:04:05Z07:00", t.CreatedAt) + t.CreatedAt = ts.Format("January 2, 2006") + topic.Topic = append(topic.Topic, t) + } + m := structs.Map(topic) + return m +} + // Topics n/a func Topics(path string) map[string]interface{} { var topic TopicsList diff --git a/data.go b/data.go index 5e3efd2..c10bf6a 100644 --- a/data.go +++ b/data.go @@ -25,6 +25,12 @@ type Options struct { BaseDir string `json:"BaseDir"` } `json:"ace"` } `json:"options"` + Database struct { + Server string `json:"server"` + DB string `json:"db"` + User string `json:"user"` + Passwd string `json:"passwd"` + } `json:"database"` } // Contacts lists contact information @@ -48,9 +54,66 @@ type Tags struct { // TagTopics list of topics via tag type TagTopics struct { - TopicList struct { + Users []struct { + ID int `json:"id"` + Username string `json:"username"` + Name string `json:"name"` + AvatarTemplate string `json:"avatar_template"` + } `json:"users"` + PrimaryGroups []interface{} `json:"primary_groups"` + TopicList struct { + CanCreateTopic bool `json:"can_create_topic"` + Draft interface{} `json:"draft"` + DraftKey string `json:"draft_key"` + DraftSequence int `json:"draft_sequence"` + PerPage int `json:"per_page"` + TopTags []string `json:"top_tags"` + Tags []struct { + ID int `json:"id"` + Name string `json:"name"` + TopicCount int `json:"topic_count"` + Staff bool `json:"staff"` + } `json:"tags"` Topics []struct { - Slug string `json:"slug"` + ID int `json:"id"` + Title string `json:"title"` + FancyTitle string `json:"fancy_title"` + Slug string `json:"slug"` + PostsCount int `json:"posts_count"` + ReplyCount int `json:"reply_count"` + HighestPostNumber int `json:"highest_post_number"` + ImageURL string `json:"image_url"` + CreatedAt time.Time `json:"created_at"` + LastPostedAt time.Time `json:"last_posted_at"` + Bumped bool `json:"bumped"` + BumpedAt time.Time `json:"bumped_at"` + Unseen bool `json:"unseen"` + LastReadPostNumber int `json:"last_read_post_number"` + Unread int `json:"unread"` + NewPosts int `json:"new_posts"` + Pinned bool `json:"pinned"` + Unpinned interface{} `json:"unpinned"` + Visible bool `json:"visible"` + Closed bool `json:"closed"` + Archived bool `json:"archived"` + NotificationLevel int `json:"notification_level"` + Bookmarked bool `json:"bookmarked"` + Liked bool `json:"liked"` + Tags []string `json:"tags"` + Views int `json:"views"` + LikeCount int `json:"like_count"` + HasSummary bool `json:"has_summary"` + Archetype string `json:"archetype"` + LastPosterUsername string `json:"last_poster_username"` + CategoryID int `json:"category_id"` + PinnedGlobally bool `json:"pinned_globally"` + FeaturedLink interface{} `json:"featured_link"` + Posters []struct { + Extras string `json:"extras"` + Description string `json:"description"` + UserID int `json:"user_id"` + PrimaryGroupID interface{} `json:"primary_group_id"` + } `json:"posters"` } `json:"topics"` } `json:"topic_list"` }