1package main
2
3import (
4 "archive/tar"
5 "bytes"
6 "flag"
7 "fmt"
8 "html"
9 "io"
10 "io/ioutil"
11 "os"
12 "path/filepath"
13 "regexp"
14 "sort"
15 "strconv"
16 "strings"
17 "time"
18
19 "github.com/goccy/go-yaml"
20
21 "github.com/yuin/goldmark"
22 goldmark_parser "github.com/yuin/goldmark/parser"
23 goldmark_extension "github.com/yuin/goldmark/extension"
24
25 hugo_goldmark_extras "github.com/gohugoio/hugo-goldmark-extensions/extras"
26
27 "sitegen/goldmark-extensions"
28
29 html_formatter "github.com/alecthomas/chroma/v2/formatters/html"
30 "github.com/alecthomas/chroma/v2/styles"
31
32 "github.com/ulikunitz/xz"
33)
34
35type
36 Page struct {
37 Draft bool
38 Is_index bool
39 Date_started string
40 Date_created string
41 Date_published string
42 Title string
43 Description string
44 Content string
45 }
46
47func main() {
48
49 var src_dir, dest_dir, codebase_dir string
50 flag.StringVar(&src_dir, "src-dir", "", "source directory")
51 flag.StringVar(&codebase_dir, "codebase", "", "codebase directory")
52 flag.StringVar(&dest_dir, "dest-dir", "", "destination directory")
53 flag.Parse()
54 if src_dir == "" {
55 fmt.Println("Need --src-dir")
56 }
57 if dest_dir == "" {
58 fmt.Println("Need --dest-dir")
59 }
60 if src_dir == "" || dest_dir == "" {
61 os.Exit(1)
62 }
63
64 // generate content
65
66 template, err := os.ReadFile(filepath.Join(filepath.Dir(os.Args[0]), "template.html"))
67 if err != nil {
68 panic(err)
69 }
70 process_src_dir(src_dir, dest_dir, template)
71
72 // generate codebase tree
73
74 if codebase_dir != "" {
75 process_codebase(codebase_dir, filepath.Join(dest_dir, "codebase"), template)
76 }
77
78 // write code highlighting stylesheet
79
80 write_stylesheet(dest_dir)
81}
82
83func process_src_dir(src_dir, dest_dir string, template []byte) {
84
85 fmt.Println("Processing directory", src_dir)
86 entries, err := os.ReadDir(src_dir)
87 if err != nil {
88 panic(err)
89 }
90 for _, entry := range entries {
91 src_path := filepath.Join(src_dir, entry.Name())
92 dest_path := filepath.Join(dest_dir, entry.Name())
93 if entry.IsDir() {
94 process_src_dir(src_path, dest_path, template)
95 } else if filepath.Ext(dest_path) == ".myaw" {
96 dest_path = strings.TrimSuffix(dest_path, ".myaw")
97 process_file(src_path, dest_path, template)
98 }
99 }
100}
101
102func process_file(src_filename, dest_path string, template []byte) {
103
104 fmt.Println("Processing file", src_filename)
105
106 // parse MYAW file (using YAML parser for now)
107
108 yml, err := ioutil.ReadFile(src_filename)
109 if err != nil {
110 panic(err)
111 }
112
113 var page Page
114 err = yaml.Unmarshal(yml, &page)
115 if err != nil {
116 panic(err)
117 }
118
119 if page.Draft {
120 return
121 }
122
123 // substitutions
124
125 content := strings.ReplaceAll(page.Content, "{{ date_started }}", page.Date_started)
126 content = strings.ReplaceAll(content, "{{ date_created }}", page.Date_created)
127 content = strings.ReplaceAll(content, "{{ date_published }}", page.Date_published)
128 content = strings.ReplaceAll(content, "{{ title }}", page.Title)
129 content = strings.ReplaceAll(content, "{{ description }}", page.Description)
130
131 // process markdown
132
133 var content_html bytes.Buffer
134 markdown_to_html(content, &content_html)
135
136 html_page := make_page(template, []byte(page.Title), content_html.Bytes())
137
138 // write output file
139
140 var dest_filename string
141
142 if page.Is_index {
143 dest_filename = filepath.Join(filepath.Dir(dest_path), "index.html")
144 } else {
145 dest_filename = filepath.Join(dest_path, "index.html")
146 }
147 err = os.MkdirAll(filepath.Dir(dest_filename), 0o755)
148 if err != nil {
149 panic(err)
150 }
151 err = ioutil.WriteFile(dest_filename, html_page, 0o644)
152 if err != nil {
153 panic(err)
154 }
155}
156
157func markdown_to_html(markdown string, html io.Writer) {
158
159 md := goldmark.New(
160 goldmark.WithExtensions(
161 // goldmark_extension.Strikethrough, // using hugo_goldmark_extras Delete extension
162 goldmark_extension.DefinitionList,
163 goldmark_extension.Footnote,
164 goldmark_extension.Linkify,
165 goldmark_extension.Table,
166 // tables.New(),
167 goldmark_extensions.CodeBlocksExtension,
168 // images.New()
169 // goldmark_extension.NewTypographer(
170 // goldmark_extension.WithTypographicSubstitutions(toTypographicPunctuationMap(cfg.Extensions.Typographer))
171 // ),
172 // goldmark_extension.TaskList,
173 hugo_goldmark_extras.New(
174 hugo_goldmark_extras.Config{
175 Delete: hugo_goldmark_extras.DeleteConfig{Enable: true},
176 Insert: hugo_goldmark_extras.InsertConfig{Enable: true},
177 Mark: hugo_goldmark_extras.MarkConfig{Enable: true},
178 Subscript: hugo_goldmark_extras.SubscriptConfig{Enable: true},
179 Superscript: hugo_goldmark_extras.SuperscriptConfig{Enable: true},
180 },
181 ),
182 ),
183 goldmark.WithParserOptions(
184 goldmark_parser.WithBlockParsers(),
185 goldmark_parser.WithInlineParsers(),
186 goldmark_parser.WithAutoHeadingID(),
187 ),
188 goldmark.WithRendererOptions(),
189 )
190 err := md.Convert([]byte(markdown), html)
191 if err != nil {
192 panic(err)
193 }
194}
195
196func make_page(template, title, content []byte) []byte {
197
198 html_page := bytes.ReplaceAll(template, []byte("{{ title }}"), title)
199 //html_page = bytes.ReplaceAll(html_page, []byte("{{ description }})", []byte(page.Description))
200 html_page = bytes.ReplaceAll(html_page, []byte("{{ content }}"), content)
201
202 // fix links
203
204 return bytes.ReplaceAll(html_page, []byte("href=\"/"), []byte("href=\"/~petbrain/"))
205}
206
207func write_stylesheet(dest_dir string) {
208
209 css_dir := filepath.Join(dest_dir, "css")
210
211 err := os.MkdirAll(css_dir, 0o755)
212 if err != nil {
213 panic(err)
214 }
215
216 theme_css, err := os.ReadFile(filepath.Join(filepath.Dir(os.Args[0]), "theme.css"))
217 if err != nil {
218 panic(err)
219 }
220
221 output_file, err := os.Create(filepath.Join(css_dir, "theme.css"))
222 if err != nil {
223 panic(err)
224 }
225
226 defer func() {
227 err := output_file.Close()
228 if err != nil {
229 panic(err)
230 }
231 }()
232
233 output_file.Write(theme_css)
234
235 formatter := html_formatter.New(html_formatter.WithClasses(true))
236 style := styles.Get(goldmark_extensions.ChromaStyle)
237 if style == nil {
238 style = styles.Fallback
239 }
240
241 err = formatter.WriteCSS(output_file, style)
242 if err != nil {
243 panic(err)
244 }
245}
246
247type (
248 Tarball struct {
249 filename string
250 version string
251 major int
252 minor int
253 patch int
254 }
255
256 SourceFile struct {
257 name string
258 size int64
259 modtime time.Time
260 }
261)
262
263func process_codebase(src_dir, dest_dir string, template []byte) {
264
265 // collect source tarballs
266 // filename is in the form project-name-<version>.tar.xz
267 // i.e. version follows final hyphen
268
269 fmt.Println("Processing codebase", src_dir)
270 entries, err := os.ReadDir(src_dir)
271 if err != nil {
272 panic(err)
273 }
274 packages := make(map[string][]Tarball)
275 for _, entry := range entries {
276 if strings.HasSuffix(entry.Name(), ".tar.xz") {
277
278 var tarball Tarball
279 tarball.filename = entry.Name()
280
281 // parse file name
282
283 parts := strings.Split(strings.TrimSuffix(entry.Name(), ".tar.xz"), "-")
284 tarball.version = parts[len(parts) - 1]
285 ver := strings.Split(tarball.version, ".")
286 package_name := strings.Join(parts[:len(parts) - 1], "-")
287
288 tarball.major, err = strconv.Atoi(ver[0])
289 if err != nil {
290 panic(err)
291 }
292 tarball.minor, err = strconv.Atoi(ver[1])
293 if err != nil {
294 panic(err)
295 }
296 tarball.patch, err = strconv.Atoi(ver[2])
297 if err != nil {
298 panic(err)
299 }
300 tarballs, ok := packages[package_name]
301 if !ok {
302 tarballs = make([]Tarball, 0)
303 }
304 tarballs = append(tarballs, tarball)
305 packages[package_name] = tarballs
306 }
307 }
308
309 for package_name, tarballs := range packages {
310 fmt.Println(" ", package_name)
311 sort.Slice(tarballs, func(i, j int) bool {
312 if tarballs[i].major > tarballs[j].major {
313 return true
314 }
315 if tarballs[i].minor > tarballs[j].minor {
316 return true
317 }
318 if tarballs[i].patch > tarballs[j].patch {
319 return true
320 }
321 return false
322 })
323 for _, tarball := range tarballs {
324 src_path := filepath.Join(src_dir, tarball.filename)
325 dest_path := filepath.Join(dest_dir, package_name, tarball.version)
326 filelist, readme_filename, readme_md := process_source_tarball(src_path, dest_path, template)
327 make_listing_page(package_name, dest_path, filelist, readme_filename, readme_md, template)
328 }
329 // symlink to the latest version
330 dest_path := filepath.Join(dest_dir, package_name, "latest")
331 os.Remove(dest_path)
332 os.Symlink(tarballs[0].version, dest_path)
333 }
334}
335
336func process_source_tarball(src_filename, dest_dir string, template []byte) ([]SourceFile, string, string) {
337
338 fmt.Println(" ", src_filename)
339
340 var filelist []SourceFile
341 var readme_filename string = ""
342 var readme_md string = ""
343
344 f, err := os.Open(src_filename)
345 if err != nil {
346 panic(err)
347 }
348 xz_reader, err := xz.NewReader(f)
349 if err != nil {
350 panic(err)
351 }
352 tar_reader := tar.NewReader(xz_reader)
353 for {
354 hdr, err := tar_reader.Next()
355 if err == io.EOF {
356 break
357 }
358 if err != nil {
359 panic(err)
360 }
361 if (hdr.Typeflag == tar.TypeReg) || (tar.TypeRegA == hdr.Typeflag) {
362 fmt.Println(hdr.Name,strings.SplitN(hdr.Name, "/", 2))
363 filename := strings.SplitN(hdr.Name, "/", 2)[1] // XXX not checking file path, just cut first dir
364 filelist = append(filelist, SourceFile{filename, hdr.Size, hdr.ModTime})
365 content, err := ioutil.ReadAll(tar_reader)
366 if err != nil {
367 panic(err)
368 }
369 if strings.HasSuffix(filename, "README.md") {
370 if (readme_filename == "") || (len(filename) < len(readme_filename)) {
371 readme_filename = filename
372 readme_md = string(content)
373 }
374 }
375 process_source_file(filename, dest_dir, string(content), template)
376 }
377 }
378 f.Close()
379 return filelist, readme_filename, readme_md
380}
381
382func process_source_file(filename, dest_dir, content string, template []byte) {
383
384 var highlighted_code bytes.Buffer
385
386 if strings.HasSuffix(filename, ".md") {
387 markdown_to_html(content, &highlighted_code)
388 } else {
389 goldmark_extensions.Highlight(filename, goldmark_extensions.Chomp(content), &highlighted_code, true)
390 }
391
392 html_page := make_page(template, []byte(filename), highlighted_code.Bytes())
393
394 // fix relative links
395
396 var re = regexp.MustCompile(`href="([^/].*?)"`)
397 html_page = re.ReplaceAllFunc(html_page, func(m []byte) []byte {
398 if bytes.HasPrefix(m, []byte("href=\"http://")) || bytes.HasPrefix(m, []byte("href=\"https://")) {
399 return m
400 } else {
401 return append([]byte("href=\"../"), m[6:]...)
402 }
403 })
404
405 // write output file
406
407 dest_path := filepath.Join(dest_dir, filename)
408 os.MkdirAll(dest_path, 0o755)
409
410 dest_filename := filepath.Join(dest_path, "index.html")
411
412 err := ioutil.WriteFile(dest_filename, html_page, 0o644)
413 if err != nil {
414 panic(err)
415 }
416}
417
418func make_listing_page(package_name, dest_dir string, filelist []SourceFile, readme_filename, readme_md string, template []byte) {
419
420 sort.Slice(filelist, func(i, j int) bool {
421 return strings.ToLower(filelist[i].name) < strings.ToLower(filelist[j].name)
422 })
423
424 var listing bytes.Buffer
425
426 listing.WriteString("<table class=\"listing\"><thead><tr><th>File</th><th>Size</th><th>Modified</th></thead><tbody>")
427 for _, entry := range filelist {
428 listing.WriteString("<tr><td><a href=\"")
429 listing.WriteString(entry.name)
430 listing.WriteString("\">")
431 listing.WriteString(html.EscapeString(entry.name))
432 listing.WriteString("</a></td><td>")
433 listing.WriteString(fmt.Sprintf("%d", entry.size))
434 listing.WriteString("</td><td>")
435 listing.WriteString(fmt.Sprintf("%d-%02d-%02d %02d:%02d",
436 entry.modtime.Year(), entry.modtime.Month(), entry.modtime.Day(),
437 entry.modtime.Hour(), entry.modtime.Minute()))
438 listing.WriteString("</td></tr>")
439 }
440 listing.WriteString("</tbody></table>")
441
442 // append readme
443
444 if readme_filename != "" {
445 markdown_to_html(readme_md, &listing)
446 }
447
448 html_page := make_page(template, []byte(package_name), listing.Bytes())
449
450 // write output file
451
452 os.MkdirAll(dest_dir, 0o755)
453 dest_filename := filepath.Join(dest_dir, "index.html")
454
455 err := ioutil.WriteFile(dest_filename, html_page, 0o644)
456 if err != nil {
457 panic(err)
458 }
459}