mirror of
https://github.com/owncloud/ocis
synced 2026-04-25 17:25:21 +02:00
feat: ci, weekly pipeline report, 2 weeks window to gather drone pipeline logs before expiry (#12053)
This commit is contained in:
Michal Klos
32
.github/workflows/weekly_pipeline_report.yml
vendored
Normal file
32
.github/workflows/weekly_pipeline_report.yml
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
name: Weekly Pipeline Report
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
branches:
|
||||
- feat/ci-report-tool
|
||||
schedule:
|
||||
- cron: "0 8 * * 1" # Every Monday at 8:00 AM UTC
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
generate-report:
|
||||
name: Generate Pipeline Report
|
||||
runs-on: ubuntu-latest
|
||||
container: golang:1.25-alpine
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run CI Reporter
|
||||
working-directory: tools/ci-reporter
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# Review the pipelines in last 14 days before the dorne logs expire
|
||||
run: |
|
||||
GOWORK=off go run ./cmd/list-failed-commits \
|
||||
--github-token "$GITHUB_TOKEN" \
|
||||
--since 14d \
|
||||
--max-commits 9999999
|
||||
379
tools/ci-reporter/cmd/list-failed-commits/main.go
Normal file
379
tools/ci-reporter/cmd/list-failed-commits/main.go
Normal file
@@ -0,0 +1,379 @@
|
||||
// Usage examples:
|
||||
// go run ./cmd/list-failed-commits --max-commits 20
|
||||
// go run ./cmd/list-failed-commits --max-commits 50 --out failed.json
|
||||
// go run ./cmd/list-failed-commits --github-token $GITHUB_TOKEN --max-commits 100
|
||||
// go run ./cmd/list-failed-commits --repo owncloud/ocis --branch master --max-commits 20
|
||||
// go run ./cmd/list-failed-commits --since 7d --max-commits 50
|
||||
// go run ./cmd/list-failed-commits --since 30d --out failed.json
|
||||
// go run ./cmd/list-failed-commits --since 2026-01-01 --max-commits 100
|
||||
// go run ./cmd/list-failed-commits --since 2026-02-15 --out recent.json
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/droneextractor"
|
||||
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/githubextractor"
|
||||
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/util"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultRepo = "owncloud/ocis"
|
||||
defaultBranch = "master"
|
||||
requestDelay = 1 * time.Second
|
||||
defaultLimit = 20
|
||||
defaultUnlimited = 999999999
|
||||
)
|
||||
|
||||
type RunConfig struct {
|
||||
Repo string
|
||||
Branch string
|
||||
MaxCommits int
|
||||
SinceRaw string
|
||||
SinceParsed string
|
||||
OutFile string
|
||||
Token string
|
||||
Command string
|
||||
FetchFailedPipelineHistory bool
|
||||
}
|
||||
|
||||
type FailureCount struct {
|
||||
StageName string `json:"stage_name"`
|
||||
StepName string `json:"step_name"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
type DateRange struct {
|
||||
StartDate string `json:"start_date"`
|
||||
EndDate string `json:"end_date"`
|
||||
}
|
||||
|
||||
type Raport struct {
|
||||
Args string `json:"args"`
|
||||
Repo string `json:"repo"`
|
||||
Branch string `json:"branch"`
|
||||
DateRange DateRange `json:"date_range"`
|
||||
TotalCommits int `json:"total_commits"`
|
||||
FailedCommits int `json:"failed_commits"`
|
||||
Percentage float64 `json:"percentage"`
|
||||
}
|
||||
|
||||
type RaportMetadata struct {
|
||||
Args string
|
||||
Repo string
|
||||
Branch string
|
||||
DateRange DateRange
|
||||
TotalCommits int
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Raport *Raport `json:"raport"`
|
||||
Count []FailureCount `json:"count"`
|
||||
Pipelines []FailedCommit `json:"pipelines"`
|
||||
}
|
||||
|
||||
type FailedCommit struct {
|
||||
PR int `json:"pr,omitempty"`
|
||||
SHA string `json:"sha"`
|
||||
Date string `json:"date"`
|
||||
Subject string `json:"subject"`
|
||||
HTMLURL string `json:"html_url"`
|
||||
CombinedState string `json:"combined_state"`
|
||||
FailedContexts []StatusContext `json:"failed_contexts"`
|
||||
PipelineInfo *droneextractor.PipelineInfo `json:"pipeline_info,omitempty"`
|
||||
FailedPipelineHistory []droneextractor.PipelineInfoSummary `json:"failed_pipeline_history,omitempty"`
|
||||
PrPipelineHistory []droneextractor.PipelineInfoSummary `json:"pr_pipeline_history,omitempty"`
|
||||
}
|
||||
|
||||
type StatusContext struct {
|
||||
Context string `json:"context"`
|
||||
State string `json:"state"`
|
||||
TargetURL string `json:"target_url"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
repo := flag.String("repo", defaultRepo, "GitHub repository (owner/repo)")
|
||||
branch := flag.String("branch", defaultBranch, "Branch name")
|
||||
maxCommits := flag.Int("max-commits", 0, "Maximum commits to check (default: 20 if no --since, unlimited if --since set)")
|
||||
since := flag.String("since", "", "Date filter: YYYY-MM-DD, RFC3339 timestamp, or 'Nd' for last N days (e.g., '7d', '30d')")
|
||||
outFile := flag.String("out", "", "Output file path (default: stdout)")
|
||||
githubToken := flag.String("github-token", "", "GitHub token (optional; falls back to env GITHUB_TOKEN)")
|
||||
failedPipelineHistory := flag.Bool("failed_pipeline_history", false, "if set, fetch failed_pipeline_history per commit (extra Drone API calls)")
|
||||
flag.Parse()
|
||||
|
||||
token := strings.TrimSpace(*githubToken)
|
||||
if token == "" {
|
||||
token = strings.TrimSpace(os.Getenv("GITHUB_TOKEN"))
|
||||
}
|
||||
|
||||
if token != "" {
|
||||
fmt.Fprintf(os.Stderr, "Using GitHub token\n")
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "No token provided, unauthenticated, 60 req/hr limit\n")
|
||||
}
|
||||
|
||||
command := strings.Join(os.Args[1:], " ")
|
||||
command = util.RedactGitHubToken(command)
|
||||
|
||||
parsedSince, err := util.ParseSinceFlag(*since)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error parsing --since flag: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Determine final max commits based on git-like UX:
|
||||
// - No flags: default to 20
|
||||
// - --since only: unlimited (all in range)
|
||||
// - --max-commits only: use explicit value
|
||||
// - Both: use explicit value (both filters apply)
|
||||
finalMaxCommits := *maxCommits
|
||||
if finalMaxCommits == 0 {
|
||||
if *since != "" {
|
||||
finalMaxCommits = defaultUnlimited // Unlimited when --since is used
|
||||
} else {
|
||||
finalMaxCommits = defaultLimit // Default for no flags
|
||||
}
|
||||
}
|
||||
|
||||
config := RunConfig{
|
||||
Repo: *repo,
|
||||
Branch: *branch,
|
||||
MaxCommits: finalMaxCommits,
|
||||
SinceRaw: *since,
|
||||
SinceParsed: parsedSince,
|
||||
OutFile: *outFile,
|
||||
Token: token,
|
||||
Command: command,
|
||||
FetchFailedPipelineHistory: *failedPipelineHistory,
|
||||
}
|
||||
|
||||
if err := run(config); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(cfg RunConfig) error {
|
||||
// Configure Transport to prevent stale connection reuse during long pagination
|
||||
transport := &http.Transport{
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
DisableKeepAlives: false,
|
||||
MaxIdleConnsPerHost: 2,
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: 60 * time.Second, // Increased for long-running pagination
|
||||
Transport: transport,
|
||||
}
|
||||
githubExtractor := githubextractor.NewExtractor(client, cfg.Token)
|
||||
droneExtractor := droneextractor.NewExtractor(client)
|
||||
|
||||
// Build scanning message
|
||||
var scanMsg string
|
||||
if cfg.SinceRaw != "" {
|
||||
scanMsg = fmt.Sprintf("Scanning commits since %s from %s/%s", cfg.SinceRaw, cfg.Repo, cfg.Branch)
|
||||
} else {
|
||||
scanMsg = fmt.Sprintf("Scanning up to %d commits from %s/%s", cfg.MaxCommits, cfg.Repo, cfg.Branch)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "%s\n", scanMsg)
|
||||
commits, err := githubExtractor.GetCommits(cfg.Repo, cfg.Branch, cfg.MaxCommits, cfg.SinceParsed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetching commits: %w", err)
|
||||
}
|
||||
|
||||
var failedCommits []FailedCommit
|
||||
for _, commit := range commits {
|
||||
status, err := githubExtractor.GetCommitStatus(cfg.Repo, commit.SHA)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "⚠ %s: %v\n", commit.SHA[:7], err)
|
||||
time.Sleep(requestDelay)
|
||||
continue
|
||||
}
|
||||
|
||||
if status.State == "failure" || status.State == "error" {
|
||||
var failedCtxs []StatusContext
|
||||
var pipelineInfo *droneextractor.PipelineInfo
|
||||
|
||||
for _, s := range status.Statuses {
|
||||
if s.State != "success" {
|
||||
// Skip Drone entries - they'll be in pipeline_info instead
|
||||
if droneExtractor.IsDroneURL(s.TargetURL) {
|
||||
info, err := droneExtractor.Extract(s.TargetURL)
|
||||
if err == nil {
|
||||
pipelineInfo = info
|
||||
}
|
||||
// Don't add to failed_contexts to avoid redundancy
|
||||
continue
|
||||
}
|
||||
failedCtxs = append(failedCtxs, toStatusContext(s))
|
||||
}
|
||||
}
|
||||
// TODO: Refactor redundant field assignment - prPipelineHistory always assigned, failedPipelineHistory conditional
|
||||
var failedPipelineHistory []droneextractor.PipelineInfoSummary
|
||||
var prPipelineHistory []droneextractor.PipelineInfoSummary
|
||||
var prNumber int
|
||||
var headRef string
|
||||
|
||||
// Get PR for this commit (master commit -> PR ID -> PR branch)
|
||||
prs, err := githubExtractor.GetPRsForCommit(cfg.Repo, commit.SHA)
|
||||
if err == nil && len(prs) > 0 {
|
||||
prNumber = prs[0].Number
|
||||
headRef = prs[0].Head.Ref
|
||||
|
||||
// Get failed builds by PR branch (black box: all logic in droneextractor)
|
||||
failedBuilds, err := droneExtractor.GetFailedBuildsByPRBranch(droneextractor.BaseURL, cfg.Repo, headRef)
|
||||
if err == nil {
|
||||
prPipelineHistory = failedBuilds
|
||||
if cfg.FetchFailedPipelineHistory {
|
||||
failedPipelineHistory = failedBuilds
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize empty slice if flag is set but no PR/builds found
|
||||
if cfg.FetchFailedPipelineHistory && failedPipelineHistory == nil {
|
||||
failedPipelineHistory = []droneextractor.PipelineInfoSummary{}
|
||||
}
|
||||
|
||||
subject := util.ExtractSubject(commit.Commit.Message)
|
||||
failedCommits = append(failedCommits, FailedCommit{
|
||||
PR: prNumber,
|
||||
SHA: commit.SHA,
|
||||
Date: commit.Commit.Author.Date,
|
||||
Subject: subject,
|
||||
HTMLURL: commit.HTMLURL,
|
||||
CombinedState: status.State,
|
||||
FailedContexts: failedCtxs,
|
||||
PipelineInfo: pipelineInfo,
|
||||
FailedPipelineHistory: failedPipelineHistory,
|
||||
PrPipelineHistory: prPipelineHistory,
|
||||
})
|
||||
|
||||
// Single line output: ✗ sha subject -> PR #num (branch: name)
|
||||
if prNumber > 0 && headRef != "" {
|
||||
fmt.Fprintf(os.Stderr, "✗ %s %s -> PR #%d (branch: %s)\n", commit.SHA[:7], subject, prNumber, headRef)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "✗ %s %s\n", commit.SHA[:7], subject)
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(requestDelay)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\nResult: %d failed / %d total\n", len(failedCommits), len(commits))
|
||||
|
||||
var minDate, maxDate string
|
||||
for _, c := range commits {
|
||||
date := c.Commit.Author.Date
|
||||
if minDate == "" || date < minDate {
|
||||
minDate = date
|
||||
}
|
||||
if maxDate == "" || date > maxDate {
|
||||
maxDate = date
|
||||
}
|
||||
}
|
||||
|
||||
metadata := RaportMetadata{
|
||||
Args: cfg.Command,
|
||||
Repo: cfg.Repo,
|
||||
Branch: cfg.Branch,
|
||||
DateRange: DateRange{StartDate: minDate, EndDate: maxDate},
|
||||
TotalCommits: len(commits),
|
||||
}
|
||||
|
||||
report := generateReport(failedCommits, metadata)
|
||||
|
||||
var w io.Writer = os.Stdout
|
||||
if cfg.OutFile != "" {
|
||||
f, err := os.Create(cfg.OutFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating output file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
w = f
|
||||
fmt.Fprintf(os.Stderr, "Writing JSON to %s...\n", cfg.OutFile)
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(report); err != nil {
|
||||
return fmt.Errorf("encoding JSON: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func toStatusContext(ctx githubextractor.StatusContext) StatusContext {
|
||||
return StatusContext{
|
||||
Context: ctx.Context,
|
||||
State: ctx.State,
|
||||
TargetURL: ctx.TargetURL,
|
||||
Description: ctx.Description,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
func generateReport(failedCommits []FailedCommit, metadata RaportMetadata) *Report {
|
||||
countMap := make(map[string]int)
|
||||
|
||||
for _, commit := range failedCommits {
|
||||
if commit.PipelineInfo == nil {
|
||||
continue
|
||||
}
|
||||
for _, stage := range commit.PipelineInfo.PipelineStages {
|
||||
for _, step := range stage.Steps {
|
||||
if step.Status == "failure" {
|
||||
key := stage.StageName + "/" + step.StepName
|
||||
countMap[key]++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var counts []FailureCount
|
||||
for key, count := range countMap {
|
||||
parts := strings.Split(key, "/")
|
||||
if len(parts) >= 2 {
|
||||
counts = append(counts, FailureCount{
|
||||
StageName: parts[0],
|
||||
StepName: strings.Join(parts[1:], "/"),
|
||||
Count: count,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(counts, func(i, j int) bool {
|
||||
return counts[i].Count > counts[j].Count
|
||||
})
|
||||
|
||||
percentage := 0.0
|
||||
if metadata.TotalCommits > 0 {
|
||||
percentage = float64(len(failedCommits)) / float64(metadata.TotalCommits) * 100.0
|
||||
}
|
||||
|
||||
raport := &Raport{
|
||||
Args: metadata.Args,
|
||||
Repo: metadata.Repo,
|
||||
Branch: metadata.Branch,
|
||||
DateRange: metadata.DateRange,
|
||||
TotalCommits: metadata.TotalCommits,
|
||||
FailedCommits: len(failedCommits),
|
||||
Percentage: percentage,
|
||||
}
|
||||
|
||||
return &Report{
|
||||
Raport: raport,
|
||||
Count: counts,
|
||||
Pipelines: failedCommits,
|
||||
}
|
||||
}
|
||||
5
tools/ci-reporter/go.mod
Normal file
5
tools/ci-reporter/go.mod
Normal file
@@ -0,0 +1,5 @@
|
||||
module github.com/owncloud/ocis/v2/tools/ci-reporter
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require golang.org/x/net v0.50.0 // indirect
|
||||
2
tools/ci-reporter/go.sum
Normal file
2
tools/ci-reporter/go.sum
Normal file
@@ -0,0 +1,2 @@
|
||||
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
|
||||
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
|
||||
631
tools/ci-reporter/pkg/droneextractor/extractor.go
Normal file
631
tools/ci-reporter/pkg/droneextractor/extractor.go
Normal file
@@ -0,0 +1,631 @@
|
||||
package droneextractor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
const (
|
||||
// BaseURL is the default Drone instance URL
|
||||
BaseURL = "https://drone.owncloud.com"
|
||||
)
|
||||
|
||||
type Extractor struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewExtractor(client *http.Client) *Extractor {
|
||||
return &Extractor{client: client}
|
||||
}
|
||||
|
||||
// IsDroneURL checks if a URL is a Drone CI URL
|
||||
func (e *Extractor) IsDroneURL(url string) bool {
|
||||
return strings.Contains(url, "drone.owncloud.com")
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(buildURL string) (*PipelineInfo, error) {
|
||||
apiURL := e.buildURLToAPIURL(buildURL)
|
||||
if apiURL == "" {
|
||||
return nil, fmt.Errorf("invalid build URL: %s", buildURL)
|
||||
}
|
||||
|
||||
buildData, err := e.fetchDroneAPI(apiURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching Drone API: %w", err)
|
||||
}
|
||||
|
||||
stages := e.parseStagesFromAPI(buildData, buildURL)
|
||||
|
||||
var durationMinutes float64
|
||||
if buildData.Finished > 0 && buildData.Started > 0 {
|
||||
durationMinutes = float64(buildData.Finished-buildData.Started) / 60.0
|
||||
}
|
||||
|
||||
return &PipelineInfo{
|
||||
BuildURL: buildURL,
|
||||
Started: buildData.Started,
|
||||
Finished: buildData.Finished,
|
||||
DurationMinutes: durationMinutes,
|
||||
PipelineStages: stages,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) buildURLToAPIURL(buildURL string) string {
|
||||
if !strings.Contains(buildURL, "drone.owncloud.com") {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(buildURL, "/")
|
||||
if len(parts) < 5 {
|
||||
return ""
|
||||
}
|
||||
repo := parts[3] + "/" + parts[4]
|
||||
buildNum := parts[5]
|
||||
return fmt.Sprintf("https://drone.owncloud.com/api/repos/%s/builds/%s", repo, buildNum)
|
||||
}
|
||||
|
||||
func (e *Extractor) fetchDroneAPI(url string) (*droneBuildResponse, error) {
|
||||
resp, err := e.client.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var buildData droneBuildResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&buildData); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &buildData, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) parseStagesFromAPI(buildData *droneBuildResponse, buildURL string) []PipelineStage {
|
||||
var stages []PipelineStage
|
||||
|
||||
for _, apiStage := range buildData.Stages {
|
||||
normalizedStatus := e.normalizeStatus(apiStage.Status)
|
||||
if normalizedStatus == "success" || normalizedStatus == "skipped" {
|
||||
continue
|
||||
}
|
||||
|
||||
stage := PipelineStage{
|
||||
StageNumber: apiStage.Number,
|
||||
StageName: apiStage.Name,
|
||||
Status: normalizedStatus,
|
||||
Steps: []PipelineStep{},
|
||||
}
|
||||
|
||||
for _, apiStep := range apiStage.Steps {
|
||||
normalizedStepStatus := e.normalizeStatus(apiStep.Status)
|
||||
if normalizedStepStatus == "skipped" {
|
||||
continue
|
||||
}
|
||||
stepURL := fmt.Sprintf("%s/%d/%d", buildURL, apiStage.Number, apiStep.Number)
|
||||
step := PipelineStep{
|
||||
StepNumber: apiStep.Number,
|
||||
StepName: apiStep.Name,
|
||||
Status: normalizedStepStatus,
|
||||
URL: stepURL,
|
||||
}
|
||||
|
||||
if normalizedStepStatus == "failure" {
|
||||
logs, err := e.fetchStepLogs(stepURL, buildURL, apiStage.Number, apiStep.Number)
|
||||
if err == nil {
|
||||
lines := strings.Split(logs, "\n")
|
||||
if len(lines) > 500 && (strings.Contains(logs, "--- Failed scenarios:") || strings.Contains(logs, "Scenario:")) {
|
||||
logs = e.extractBehatFailures(logs)
|
||||
}
|
||||
step.Logs = logs
|
||||
}
|
||||
}
|
||||
|
||||
stage.Steps = append(stage.Steps, step)
|
||||
}
|
||||
|
||||
stages = append(stages, stage)
|
||||
}
|
||||
|
||||
return stages
|
||||
}
|
||||
|
||||
func (e *Extractor) normalizeStatus(status string) string {
|
||||
switch strings.ToLower(status) {
|
||||
case "success", "passed":
|
||||
return "success"
|
||||
case "failure", "failed", "error":
|
||||
return "failure"
|
||||
case "running", "pending", "started":
|
||||
return "running"
|
||||
case "skipped":
|
||||
return "skipped"
|
||||
default:
|
||||
return status
|
||||
}
|
||||
}
|
||||
|
||||
// droneBuildResponse matches the structure returned by Drone API:
|
||||
// GET /api/repos/{owner}/{repo}/builds/{buildNumber}
|
||||
// Verified against: https://drone.owncloud.com/api/repos/owncloud/ocis/builds/51629
|
||||
type droneBuildResponse struct {
|
||||
Started int64 `json:"started"`
|
||||
Finished int64 `json:"finished"`
|
||||
Stages []droneStage `json:"stages"`
|
||||
}
|
||||
|
||||
type droneStage struct {
|
||||
Number int `json:"number"` // Stage number (1, 2, 3, ...)
|
||||
Name string `json:"name"` // Stage name (e.g., "API-wopi", "coding-standard-php8.4")
|
||||
Status string `json:"status"` // "success", "failure", "running", etc.
|
||||
Steps []droneStep `json:"steps"` // Array of steps in this stage
|
||||
}
|
||||
|
||||
type droneStep struct {
|
||||
Number int `json:"number"` // Step number within stage
|
||||
Name string `json:"name"` // Step name (e.g., "clone", "test-acceptance-api")
|
||||
Status string `json:"status"` // "success", "failure", "skipped", etc.
|
||||
}
|
||||
|
||||
type droneLogEntry struct {
|
||||
Pos int `json:"pos"`
|
||||
Out string `json:"out"`
|
||||
Time int64 `json:"time"`
|
||||
}
|
||||
|
||||
type BuildInfo struct {
|
||||
BuildNumber int `json:"build_number"`
|
||||
Status string `json:"status"` // "success", "failure", "error"
|
||||
Started int64 `json:"started"`
|
||||
Finished int64 `json:"finished"`
|
||||
CommitSHA string `json:"commit_sha"`
|
||||
}
|
||||
|
||||
type droneBuildListItem struct {
|
||||
Number int `json:"number"`
|
||||
Status string `json:"status"`
|
||||
Event string `json:"event"` // "pull_request", "push", etc.
|
||||
Source string `json:"source"` // Source branch for PR builds
|
||||
Target string `json:"target"` // Target branch for PR builds
|
||||
After string `json:"after"` // Commit SHA
|
||||
Started int64 `json:"started"`
|
||||
Finished int64 `json:"finished"`
|
||||
}
|
||||
|
||||
func (e *Extractor) fetchStepLogs(stepURL, buildURL string, stageNum, stepNum int) (string, error) {
|
||||
apiLogsURL := e.buildLogsAPIURL(buildURL, stageNum, stepNum)
|
||||
if apiLogsURL != "" {
|
||||
logs, err := e.fetchLogsFromAPI(apiLogsURL)
|
||||
if err == nil {
|
||||
return logs, nil
|
||||
}
|
||||
if err != nil && !strings.Contains(err.Error(), "401") && !strings.Contains(err.Error(), "403") {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
return e.fetchLogsFromHTML(stepURL)
|
||||
}
|
||||
|
||||
func (e *Extractor) buildLogsAPIURL(buildURL string, stageNum, stepNum int) string {
|
||||
if !strings.Contains(buildURL, "drone.owncloud.com") {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(buildURL, "/")
|
||||
if len(parts) < 5 {
|
||||
return ""
|
||||
}
|
||||
repo := parts[3] + "/" + parts[4]
|
||||
buildNum := parts[5]
|
||||
return fmt.Sprintf("https://drone.owncloud.com/api/repos/%s/builds/%s/logs/%d/%d", repo, buildNum, stageNum, stepNum)
|
||||
}
|
||||
|
||||
func (e *Extractor) fetchLogsFromAPI(apiURL string) (string, error) {
|
||||
resp, err := e.client.Get(apiURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var logEntries []droneLogEntry
|
||||
if err := json.NewDecoder(resp.Body).Decode(&logEntries); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var logs strings.Builder
|
||||
for _, entry := range logEntries {
|
||||
logs.WriteString(entry.Out)
|
||||
}
|
||||
|
||||
return logs.String(), nil
|
||||
}
|
||||
|
||||
func (e *Extractor) fetchLogsFromHTML(stepURL string) (string, error) {
|
||||
resp, err := e.client.Get(stepURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return e.extractLogsFromHTML(string(body))
|
||||
}
|
||||
|
||||
func (e *Extractor) extractLogsFromHTML(htmlContent string) (string, error) {
|
||||
doc, err := html.Parse(strings.NewReader(htmlContent))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var logs strings.Builder
|
||||
var extractText func(*html.Node)
|
||||
extractText = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
if n.Data == "pre" || n.Data == "code" {
|
||||
text := e.getTextContent(n)
|
||||
if text != "" {
|
||||
logs.WriteString(text)
|
||||
logs.WriteString("\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
extractText(c)
|
||||
}
|
||||
}
|
||||
|
||||
extractText(doc)
|
||||
return strings.TrimSpace(logs.String()), nil
|
||||
}
|
||||
|
||||
func (e *Extractor) getTextContent(n *html.Node) string {
|
||||
var text strings.Builder
|
||||
var collectText func(*html.Node)
|
||||
collectText = func(n *html.Node) {
|
||||
if n.Type == html.TextNode {
|
||||
text.WriteString(n.Data)
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
collectText(c)
|
||||
}
|
||||
}
|
||||
collectText(n)
|
||||
return text.String()
|
||||
}
|
||||
|
||||
func (e *Extractor) extractBehatFailures(logs string) string {
|
||||
var result strings.Builder
|
||||
|
||||
// ANSI red code pattern
|
||||
redCodePattern := regexp.MustCompile(`\x1b\[31m|\033\[31m|\[31m`)
|
||||
ansiStripPattern := regexp.MustCompile(`\x1b\[[0-9;]*m|\033\[[0-9;]*m`)
|
||||
|
||||
// Split by Scenario: or Scenario Outline: (case-insensitive)
|
||||
// Use regex to find all scenario starts while preserving the marker
|
||||
scenarioPattern := regexp.MustCompile(`(?i)(Scenario(?:\s+Outline)?:\s*)`)
|
||||
matches := scenarioPattern.FindAllStringIndex(logs, -1)
|
||||
|
||||
if len(matches) == 0 {
|
||||
// No scenarios found, try to extract just "--- Failed scenarios:" section
|
||||
return e.extractFailedScenariosSection(logs, ansiStripPattern)
|
||||
}
|
||||
|
||||
// Extract each scenario section
|
||||
for i, match := range matches {
|
||||
start := match[0]
|
||||
end := len(logs)
|
||||
if i+1 < len(matches) {
|
||||
end = matches[i+1][0]
|
||||
}
|
||||
|
||||
scenarioSection := logs[start:end]
|
||||
|
||||
// Check if this section contains ANSI red codes
|
||||
if redCodePattern.MatchString(scenarioSection) {
|
||||
// Strip ANSI codes and add to result
|
||||
cleaned := ansiStripPattern.ReplaceAllString(scenarioSection, "")
|
||||
if result.Len() > 0 {
|
||||
result.WriteString("\n\n")
|
||||
}
|
||||
result.WriteString(cleaned)
|
||||
}
|
||||
}
|
||||
|
||||
// Extract "--- Failed scenarios:" section
|
||||
failedScenariosSection := e.extractFailedScenariosSection(logs, ansiStripPattern)
|
||||
if failedScenariosSection != "" {
|
||||
if result.Len() > 0 {
|
||||
result.WriteString("\n\n")
|
||||
}
|
||||
result.WriteString(failedScenariosSection)
|
||||
}
|
||||
|
||||
extracted := strings.TrimSpace(result.String())
|
||||
if extracted == "" {
|
||||
// If extraction failed, return original logs
|
||||
return logs
|
||||
}
|
||||
|
||||
return extracted
|
||||
}
|
||||
|
||||
func (e *Extractor) extractFailedScenariosSection(logs string, ansiStripPattern *regexp.Regexp) string {
|
||||
failedScenariosMarker := "--- Failed scenarios:"
|
||||
if idx := strings.Index(logs, failedScenariosMarker); idx != -1 {
|
||||
failedScenariosSection := logs[idx:]
|
||||
cleaned := ansiStripPattern.ReplaceAllString(failedScenariosSection, "")
|
||||
return cleaned
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetBuildsByCommitSHA queries Drone API for all builds matching a specific commit SHA
|
||||
func (e *Extractor) GetBuildsByCommitSHA(baseURL, repo, commitSHA string) ([]BuildInfo, error) {
|
||||
var allBuilds []BuildInfo
|
||||
page := 1
|
||||
maxPages := 200 // Safety limit (25 items per page = 5000 builds max)
|
||||
|
||||
for page <= maxPages {
|
||||
url := fmt.Sprintf("%s/api/repos/%s/builds?page=%d", baseURL, repo, page)
|
||||
resp, err := e.client.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching build list: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var buildList []droneBuildListItem
|
||||
if err := json.NewDecoder(resp.Body).Decode(&buildList); err != nil {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("decoding build list: %w", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
// Stop when we get an empty page
|
||||
if len(buildList) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Filter builds by commit SHA
|
||||
for _, item := range buildList {
|
||||
if item.After == commitSHA {
|
||||
allBuilds = append(allBuilds, BuildInfo{
|
||||
BuildNumber: item.Number,
|
||||
Status: item.Status,
|
||||
Started: item.Started,
|
||||
Finished: item.Finished,
|
||||
CommitSHA: item.After,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
page++
|
||||
}
|
||||
|
||||
// Sort by build number (ascending = chronological)
|
||||
sort.Slice(allBuilds, func(i, j int) bool {
|
||||
return allBuilds[i].BuildNumber < allBuilds[j].BuildNumber
|
||||
})
|
||||
|
||||
return allBuilds, nil
|
||||
}
|
||||
|
||||
// GetBuildsByPRBranch queries Drone API for all builds matching PR event and source branch
|
||||
func (e *Extractor) GetBuildsByPRBranch(baseURL, repo, sourceBranch string) ([]BuildInfo, error) {
|
||||
var allBuilds []BuildInfo
|
||||
page := 1
|
||||
maxPages := 200 // Safety limit (25 items per page = 5000 builds max)
|
||||
|
||||
for page <= maxPages {
|
||||
url := fmt.Sprintf("%s/api/repos/%s/builds?page=%d", baseURL, repo, page)
|
||||
resp, err := e.client.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching build list: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var buildList []droneBuildListItem
|
||||
if err := json.NewDecoder(resp.Body).Decode(&buildList); err != nil {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("decoding build list: %w", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
// Stop when we get an empty page
|
||||
if len(buildList) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Filter builds by event="pull_request" and source branch
|
||||
for _, item := range buildList {
|
||||
if item.Event == "pull_request" && item.Source == sourceBranch {
|
||||
allBuilds = append(allBuilds, BuildInfo{
|
||||
BuildNumber: item.Number,
|
||||
Status: item.Status,
|
||||
Started: item.Started,
|
||||
Finished: item.Finished,
|
||||
CommitSHA: item.After,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
page++
|
||||
}
|
||||
|
||||
// Sort by build number (ascending = chronological)
|
||||
sort.Slice(allBuilds, func(i, j int) bool {
|
||||
return allBuilds[i].BuildNumber < allBuilds[j].BuildNumber
|
||||
})
|
||||
|
||||
return allBuilds, nil
|
||||
}
|
||||
|
||||
// GetBuildsByPushBranch queries Drone API for all builds matching push/cron events to a specific branch (e.g., master).
|
||||
// This is specifically for merged commits to main branches, not PR builds.
|
||||
// It includes both "push" events (actual merges) and "cron" events (scheduled/nightly builds).
|
||||
func (e *Extractor) GetBuildsByPushBranch(baseURL, repo, targetBranch string) ([]BuildInfo, error) {
|
||||
var allBuilds []BuildInfo
|
||||
page := 1
|
||||
maxPages := 200 // Safety limit (25 items per page = 5000 builds max)
|
||||
|
||||
for page <= maxPages {
|
||||
url := fmt.Sprintf("%s/api/repos/%s/builds?page=%d", baseURL, repo, page)
|
||||
resp, err := e.client.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching build list: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var buildList []droneBuildListItem
|
||||
if err := json.NewDecoder(resp.Body).Decode(&buildList); err != nil {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("decoding build list: %w", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
// Stop when we get an empty page
|
||||
if len(buildList) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Filter builds by event="push" or event="cron" and target branch
|
||||
// "push" = actual merges to branch
|
||||
// "cron" = scheduled/nightly builds running on the branch
|
||||
for _, item := range buildList {
|
||||
if (item.Event == "push" || item.Event == "cron") && item.Target == targetBranch {
|
||||
allBuilds = append(allBuilds, BuildInfo{
|
||||
BuildNumber: item.Number,
|
||||
Status: item.Status,
|
||||
Started: item.Started,
|
||||
Finished: item.Finished,
|
||||
CommitSHA: item.After,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
page++
|
||||
}
|
||||
|
||||
// Sort by build number (ascending = chronological)
|
||||
sort.Slice(allBuilds, func(i, j int) bool {
|
||||
return allBuilds[i].BuildNumber < allBuilds[j].BuildNumber
|
||||
})
|
||||
|
||||
return allBuilds, nil
|
||||
}
|
||||
|
||||
// extractBaseURL extracts the base URL (e.g., "https://drone.owncloud.com") from a build URL
|
||||
func extractBaseURL(buildURL string) string {
|
||||
if !strings.Contains(buildURL, "://") {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(buildURL, "/")
|
||||
if len(parts) >= 3 {
|
||||
return parts[0] + "//" + parts[2]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// BuildInfoToPipelineSummary converts a BuildInfo to PipelineInfoSummary
|
||||
func BuildInfoToPipelineSummary(build BuildInfo, baseURL, repo string) PipelineInfoSummary {
|
||||
var durationMinutes float64
|
||||
if build.Finished > 0 && build.Started > 0 {
|
||||
durationMinutes = float64(build.Finished-build.Started) / 60.0
|
||||
}
|
||||
|
||||
// URL format: https://drone.owncloud.com/owncloud/ocis/{buildNumber}
|
||||
// repo format: "owncloud/ocis"
|
||||
buildURL := fmt.Sprintf("%s/%s/%d", baseURL, repo, build.BuildNumber)
|
||||
|
||||
return PipelineInfoSummary{
|
||||
BuildURL: buildURL,
|
||||
Started: build.Started,
|
||||
Finished: build.Finished,
|
||||
DurationMinutes: durationMinutes,
|
||||
Status: build.Status,
|
||||
}
|
||||
}
|
||||
|
||||
// BuildInfoToPipelineSummaryWithStages converts a BuildInfo to PipelineInfoSummary with failed stages
|
||||
// by fetching full pipeline details from the build URL
|
||||
func (e *Extractor) BuildInfoToPipelineSummaryWithStages(build BuildInfo, baseURL, repo string) PipelineInfoSummary {
|
||||
summary := BuildInfoToPipelineSummary(build, baseURL, repo)
|
||||
|
||||
// Fetch full pipeline info to extract failed stages
|
||||
pipelineInfo, err := e.Extract(summary.BuildURL)
|
||||
if err == nil && pipelineInfo != nil {
|
||||
var failedStages []FailedStage
|
||||
for _, stage := range pipelineInfo.PipelineStages {
|
||||
if stage.Status == "failure" || stage.Status == "error" {
|
||||
failedStages = append(failedStages, FailedStage{
|
||||
StageNumber: stage.StageNumber,
|
||||
StageName: stage.StageName,
|
||||
Status: stage.Status,
|
||||
})
|
||||
}
|
||||
}
|
||||
summary.FailedStages = failedStages
|
||||
}
|
||||
|
||||
return summary
|
||||
}
|
||||
|
||||
// IsFailedBuild checks if a build status indicates failure
|
||||
func IsFailedBuild(status string) bool {
|
||||
normalized := strings.ToLower(status)
|
||||
return normalized == "failure" || normalized == "failed" || normalized == "error"
|
||||
}
|
||||
|
||||
// GetFailedBuildsByPRBranch returns failed builds for a PR branch with pipeline summaries
|
||||
func (e *Extractor) GetFailedBuildsByPRBranch(baseURL, repo, sourceBranch string) ([]PipelineInfoSummary, error) {
|
||||
builds, err := e.GetBuildsByPRBranch(baseURL, repo, sourceBranch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var failedSummaries []PipelineInfoSummary
|
||||
for _, build := range builds {
|
||||
if IsFailedBuild(build.Status) {
|
||||
summary := e.BuildInfoToPipelineSummaryWithStages(build, baseURL, repo)
|
||||
failedSummaries = append(failedSummaries, summary)
|
||||
// Rate limiting delay for API calls in BuildInfoToPipelineSummaryWithStages
|
||||
}
|
||||
}
|
||||
|
||||
return failedSummaries, nil
|
||||
}
|
||||
78
tools/ci-reporter/pkg/droneextractor/pr_builds.go
Normal file
78
tools/ci-reporter/pkg/droneextractor/pr_builds.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package droneextractor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/owncloud/ocis/v2/tools/ci-reporter/pkg/githubextractor"
|
||||
)
|
||||
|
||||
const requestDelay = 1 * time.Second
|
||||
|
||||
// FetchPRBuilds fetches all Drone builds for a PR by querying:
|
||||
// 1. Builds by PR branch (handles force-pushes)
|
||||
// 2. Builds by individual commit SHAs (fallback for merged/deleted branches)
|
||||
//
|
||||
// Returns a map of commit SHA -> builds
|
||||
func FetchPRBuilds(
|
||||
extractor *Extractor,
|
||||
baseURL string,
|
||||
repo string,
|
||||
prBranch string,
|
||||
commits []githubextractor.PRCommit,
|
||||
logWriter io.Writer,
|
||||
) (map[string][]BuildInfo, error) {
|
||||
buildsByCommit := make(map[string][]BuildInfo)
|
||||
|
||||
// Query builds by PR branch (handles force-pushes)
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, "Fetching Drone builds for PR (by event and branch)...\n")
|
||||
}
|
||||
prBuilds, err := extractor.GetBuildsByPRBranch(baseURL, repo, prBranch)
|
||||
if err != nil {
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, "⚠ Failed to get builds by branch: %v\n", err)
|
||||
}
|
||||
} else {
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, "Found %d builds for PR branch %s\n", len(prBuilds), prBranch)
|
||||
}
|
||||
// Group builds by commit SHA
|
||||
for _, build := range prBuilds {
|
||||
buildsByCommit[build.CommitSHA] = append(buildsByCommit[build.CommitSHA], build)
|
||||
}
|
||||
}
|
||||
|
||||
// Also query by current commit SHAs (for completeness)
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, "Fetching Drone builds for current commits...\n")
|
||||
}
|
||||
for i, commit := range commits {
|
||||
if _, exists := buildsByCommit[commit.SHA]; exists {
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, " [%d/%d] %s: already found via PR query\n", i+1, len(commits), commit.SHA[:7])
|
||||
}
|
||||
continue
|
||||
}
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, " [%d/%d] Fetching builds for %s...\n", i+1, len(commits), commit.SHA[:7])
|
||||
}
|
||||
builds, err := extractor.GetBuildsByCommitSHA(baseURL, repo, commit.SHA)
|
||||
if err != nil {
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, " ⚠ Failed to get builds: %v\n", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if len(builds) > 0 {
|
||||
buildsByCommit[commit.SHA] = append(buildsByCommit[commit.SHA], builds...)
|
||||
if logWriter != nil {
|
||||
fmt.Fprintf(logWriter, " Found %d builds\n", len(builds))
|
||||
}
|
||||
}
|
||||
time.Sleep(requestDelay)
|
||||
}
|
||||
|
||||
return buildsByCommit, nil
|
||||
}
|
||||
39
tools/ci-reporter/pkg/droneextractor/types.go
Normal file
39
tools/ci-reporter/pkg/droneextractor/types.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package droneextractor
|
||||
|
||||
type PipelineInfo struct {
|
||||
BuildURL string `json:"build_url"`
|
||||
Started int64 `json:"started,omitempty"` // Unix timestamp
|
||||
Finished int64 `json:"finished,omitempty"` // Unix timestamp
|
||||
DurationMinutes float64 `json:"duration_minutes,omitempty"` // Calculated: (finished - started) / 60
|
||||
PipelineStages []PipelineStage `json:"pipeline_stages"`
|
||||
}
|
||||
|
||||
type PipelineStage struct {
|
||||
StageNumber int `json:"stage_number"`
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Steps []PipelineStep `json:"steps"`
|
||||
}
|
||||
|
||||
type PipelineStep struct {
|
||||
StepNumber int `json:"step_number"`
|
||||
StepName string `json:"step_name"`
|
||||
Status string `json:"status"` // "success", "failure", "error", "running", etc.
|
||||
URL string `json:"url,omitempty"`
|
||||
Logs string `json:"logs,omitempty"` // Console logs content
|
||||
}
|
||||
|
||||
type PipelineInfoSummary struct {
|
||||
BuildURL string `json:"build_url"`
|
||||
Started int64 `json:"started,omitempty"`
|
||||
Finished int64 `json:"finished,omitempty"`
|
||||
DurationMinutes float64 `json:"duration_minutes,omitempty"`
|
||||
Status string `json:"status,omitempty"` // Overall build status
|
||||
FailedStages []FailedStage `json:"failed_stages,omitempty"`
|
||||
}
|
||||
|
||||
type FailedStage struct {
|
||||
StageNumber int `json:"stage_number"`
|
||||
StageName string `json:"stage_name"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
308
tools/ci-reporter/pkg/githubextractor/extractor.go
Normal file
308
tools/ci-reporter/pkg/githubextractor/extractor.go
Normal file
@@ -0,0 +1,308 @@
|
||||
package githubextractor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
apiBase = "https://api.github.com"
|
||||
requestDelay = 1 * time.Second
|
||||
retryDelay = 2 * time.Second
|
||||
maxRetries = 3
|
||||
)
|
||||
|
||||
type Extractor struct {
|
||||
client *http.Client
|
||||
token string
|
||||
}
|
||||
|
||||
func NewExtractor(client *http.Client, token string) *Extractor {
|
||||
return &Extractor{
|
||||
client: client,
|
||||
token: token,
|
||||
}
|
||||
}
|
||||
|
||||
type PRInfo struct {
|
||||
Number int `json:"number"`
|
||||
Head struct {
|
||||
Ref string `json:"ref"`
|
||||
SHA string `json:"sha"`
|
||||
} `json:"head"`
|
||||
Base struct {
|
||||
Ref string `json:"ref"`
|
||||
} `json:"base"`
|
||||
}
|
||||
|
||||
type PRCommit struct {
|
||||
SHA string `json:"sha"`
|
||||
Commit struct {
|
||||
Author struct {
|
||||
Date string `json:"date"`
|
||||
} `json:"author"`
|
||||
Message string `json:"message"`
|
||||
} `json:"commit"`
|
||||
}
|
||||
|
||||
type CommitListItem struct {
|
||||
SHA string `json:"sha"`
|
||||
HTMLURL string `json:"html_url"`
|
||||
Commit struct {
|
||||
Author struct {
|
||||
Date string `json:"date"`
|
||||
} `json:"author"`
|
||||
Message string `json:"message"`
|
||||
} `json:"commit"`
|
||||
}
|
||||
|
||||
type CombinedStatus struct {
|
||||
State string `json:"state"`
|
||||
Statuses []StatusContext `json:"statuses"`
|
||||
}
|
||||
|
||||
type StatusContext struct {
|
||||
Context string `json:"context"`
|
||||
State string `json:"state"`
|
||||
TargetURL string `json:"target_url"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
func (e *Extractor) GetPRInfo(repo string, prNumber int) (*PRInfo, error) {
|
||||
url := fmt.Sprintf("%s/repos/%s/pulls/%d", apiBase, repo, prNumber)
|
||||
var prInfo PRInfo
|
||||
return &prInfo, e.get(url, &prInfo)
|
||||
}
|
||||
|
||||
func (e *Extractor) GetPRCommits(repo string, prNumber int) ([]PRCommit, error) {
|
||||
url := fmt.Sprintf("%s/repos/%s/pulls/%d/commits", apiBase, repo, prNumber)
|
||||
var commits []PRCommit
|
||||
return commits, e.get(url, &commits)
|
||||
}
|
||||
|
||||
func (e *Extractor) GetCommits(repo, branch string, maxCommits int, since string) ([]CommitListItem, error) {
|
||||
var commits []CommitListItem
|
||||
page := 1
|
||||
|
||||
for len(commits) < maxCommits {
|
||||
url := fmt.Sprintf("%s/repos/%s/commits?sha=%s&per_page=100&page=%d", apiBase, repo, branch, page)
|
||||
if since != "" {
|
||||
url += "&since=" + since
|
||||
}
|
||||
|
||||
resp, err := e.request(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("API error %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var pageCommits []CommitListItem
|
||||
if err := json.NewDecoder(resp.Body).Decode(&pageCommits); err != nil {
|
||||
resp.Body.Close()
|
||||
return nil, err
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if len(pageCommits) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
commits = append(commits, pageCommits...)
|
||||
if len(commits) >= maxCommits {
|
||||
commits = commits[:maxCommits]
|
||||
break
|
||||
}
|
||||
|
||||
page++
|
||||
time.Sleep(requestDelay)
|
||||
}
|
||||
|
||||
return commits, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) GetCommitStatus(repo, sha string) (*CombinedStatus, error) {
|
||||
url := fmt.Sprintf("%s/repos/%s/commits/%s/status", apiBase, repo, sha)
|
||||
var status CombinedStatus
|
||||
return &status, e.get(url, &status)
|
||||
}
|
||||
|
||||
// GetPRsForCommit returns PRs associated with a commit (e.g. merged PR that introduced the commit).
|
||||
// See https://docs.github.com/en/rest/commits/commits#list-pull-requests-associated-with-a-commit
|
||||
func (e *Extractor) GetPRsForCommit(repo, commitSHA string) ([]PRInfo, error) {
|
||||
url := fmt.Sprintf("%s/repos/%s/commits/%s/pulls", apiBase, repo, commitSHA)
|
||||
var prs []PRInfo
|
||||
return prs, e.get(url, &prs)
|
||||
}
|
||||
|
||||
func isRetryableNetworkError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
s := strings.ToLower(err.Error())
|
||||
return strings.Contains(s, "broken pipe") ||
|
||||
strings.Contains(s, "connection reset") ||
|
||||
strings.Contains(s, "connection refused") ||
|
||||
strings.Contains(s, "EOF")
|
||||
}
|
||||
|
||||
// get makes an HTTP GET request, decodes JSON response, and handles rate limiting
|
||||
func (e *Extractor) get(url string, result any) error {
|
||||
resp, err := e.request(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("API error %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(result); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
time.Sleep(requestDelay)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Extractor) request(url string) (*http.Response, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxRetries; attempt++ {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github.v3+json")
|
||||
if e.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+e.token)
|
||||
}
|
||||
resp, err := e.client.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
if isRetryableNetworkError(err) && attempt < maxRetries-1 {
|
||||
time.Sleep(retryDelay)
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
// Pipeline represents a single Drone pipeline run
|
||||
type Pipeline struct {
|
||||
ID int `json:"id"`
|
||||
Status string `json:"status"`
|
||||
Started int64 `json:"started"`
|
||||
Finished int64 `json:"finished"`
|
||||
Duration float64 `json:"duration_minutes"`
|
||||
Trigger string `json:"trigger"`
|
||||
}
|
||||
|
||||
// OriginalCommit represents an individual commit from the PR branch before squashing
|
||||
type OriginalCommit struct {
|
||||
SHA string `json:"sha"`
|
||||
Title string `json:"title"`
|
||||
Pipelines []Pipeline `json:"pipelines,omitempty"`
|
||||
}
|
||||
|
||||
// ExtractedCommit represents a commit with PR information
|
||||
type ExtractedCommit struct {
|
||||
PR int `json:"pr"`
|
||||
SHA string `json:"sha"`
|
||||
Date string `json:"date"`
|
||||
Title string `json:"title"`
|
||||
HTMLURL string `json:"html_url"`
|
||||
Pipelines []Pipeline `json:"pipelines,omitempty"`
|
||||
OriginalCommits []OriginalCommit `json:"original_commits,omitempty"`
|
||||
}
|
||||
|
||||
// ExtractedData represents the output structure
|
||||
type ExtractedData struct {
|
||||
Commits []ExtractedCommit `json:"commits"`
|
||||
}
|
||||
|
||||
// extractSubject extracts the first line of the commit message
|
||||
func extractSubject(message string) string {
|
||||
if idx := strings.Index(message, "\n"); idx >= 0 {
|
||||
return strings.TrimSpace(message[:idx])
|
||||
}
|
||||
return strings.TrimSpace(message)
|
||||
}
|
||||
|
||||
// ExtractCommitData fetches commits and extracts them into structured format
|
||||
func (e *Extractor) ExtractCommitData(repo, branch string, maxCommits int, since string) (*ExtractedData, error) {
|
||||
commits, err := e.GetCommits(repo, branch, maxCommits, since)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching commits: %w", err)
|
||||
}
|
||||
|
||||
extractedCommits := make([]ExtractedCommit, 0, len(commits))
|
||||
originalCommitSHAs := make(map[string]bool) // Track SHAs that appear as original commits
|
||||
|
||||
for _, commit := range commits {
|
||||
title := extractSubject(commit.Commit.Message)
|
||||
|
||||
// Fetch PR information from GitHub API
|
||||
prNumber := 0
|
||||
var originalCommits []OriginalCommit
|
||||
prs, err := e.GetPRsForCommit(repo, commit.SHA)
|
||||
if err == nil && len(prs) > 0 {
|
||||
// Use the first PR number if multiple PRs are associated
|
||||
prNumber = prs[0].Number
|
||||
|
||||
// Fetch original commits from the PR before squashing
|
||||
prCommits, err := e.GetPRCommits(repo, prNumber)
|
||||
if err == nil && len(prCommits) > 0 {
|
||||
originalCommits = make([]OriginalCommit, 0, len(prCommits))
|
||||
for _, prCommit := range prCommits {
|
||||
// Skip if this is the same commit (don't self-reference)
|
||||
if prCommit.SHA == commit.SHA {
|
||||
continue
|
||||
}
|
||||
originalCommits = append(originalCommits, OriginalCommit{
|
||||
SHA: prCommit.SHA,
|
||||
Title: extractSubject(prCommit.Commit.Message),
|
||||
})
|
||||
// Track this SHA as an original commit
|
||||
originalCommitSHAs[prCommit.SHA] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extractedCommits = append(extractedCommits, ExtractedCommit{
|
||||
PR: prNumber,
|
||||
SHA: commit.SHA,
|
||||
Date: commit.Commit.Author.Date,
|
||||
Title: title,
|
||||
HTMLURL: commit.HTMLURL,
|
||||
OriginalCommits: originalCommits,
|
||||
})
|
||||
}
|
||||
|
||||
// Filter out commits that are already listed as original commits in merge commits
|
||||
// These are PR commits that appear in master history but should only show under the merge commit
|
||||
filteredCommits := make([]ExtractedCommit, 0, len(extractedCommits))
|
||||
for _, commit := range extractedCommits {
|
||||
// Keep the commit ONLY if it's NOT in the originalCommitSHAs set
|
||||
// If a commit appears in any other commit's original_commits, it should not be a top-level entry
|
||||
if !originalCommitSHAs[commit.SHA] {
|
||||
filteredCommits = append(filteredCommits, commit)
|
||||
}
|
||||
}
|
||||
|
||||
return &ExtractedData{
|
||||
Commits: filteredCommits,
|
||||
}, nil
|
||||
}
|
||||
70
tools/ci-reporter/pkg/util/util.go
Normal file
70
tools/ci-reporter/pkg/util/util.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ParseSinceFlag parses a date/time string in multiple formats:
|
||||
// - YYYY-MM-DD: absolute date (e.g., "2026-02-15")
|
||||
// - RFC3339: timestamp (e.g., "2026-02-15T10:30:00Z")
|
||||
// - Nd: relative days (e.g., "7d", "30d")
|
||||
//
|
||||
// Returns the parsed time in RFC3339 format.
|
||||
func ParseSinceFlag(since string) (string, error) {
|
||||
if since == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Try parsing as YYYY-MM-DD
|
||||
if t, err := time.Parse("2006-01-02", since); err == nil {
|
||||
return t.Format(time.RFC3339), nil
|
||||
}
|
||||
|
||||
// Try parsing as RFC3339 (backward compatibility)
|
||||
if t, err := time.Parse(time.RFC3339, since); err == nil {
|
||||
return t.Format(time.RFC3339), nil
|
||||
}
|
||||
|
||||
// Try parsing as relative days format (e.g., "7d", "30d")
|
||||
if strings.HasSuffix(since, "d") {
|
||||
daysStr := strings.TrimSuffix(since, "d")
|
||||
days, err := time.ParseDuration(daysStr + "h")
|
||||
if err == nil {
|
||||
// Convert days to hours (Nd = N*24h)
|
||||
hours := days.Hours()
|
||||
t := time.Now().Add(-time.Duration(hours*24) * time.Hour)
|
||||
return t.Format(time.RFC3339), nil
|
||||
}
|
||||
// Try parsing as integer
|
||||
var numDays int
|
||||
if _, err := fmt.Sscanf(daysStr, "%d", &numDays); err == nil {
|
||||
t := time.Now().AddDate(0, 0, -numDays)
|
||||
return t.Format(time.RFC3339), nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("invalid format: use YYYY-MM-DD, RFC3339, or 'Nd' (e.g., '7d')")
|
||||
}
|
||||
|
||||
// ExtractSubject extracts the first line (subject) from a commit message.
|
||||
func ExtractSubject(message string) string {
|
||||
lines := strings.Split(message, "\n")
|
||||
return strings.TrimSpace(lines[0])
|
||||
}
|
||||
|
||||
// RedactGitHubToken redacts GitHub tokens from command strings for safe logging.
|
||||
// Matches --github-token=<token> or --github-token <token> patterns.
|
||||
func RedactGitHubToken(command string) string {
|
||||
// Match --github-token=<token>
|
||||
pattern1 := regexp.MustCompile(`--github-token=[^\s]+`)
|
||||
command = pattern1.ReplaceAllString(command, "--github-token=***")
|
||||
|
||||
// Match --github-token <token>
|
||||
pattern2 := regexp.MustCompile(`--github-token\s+[^\s]+`)
|
||||
command = pattern2.ReplaceAllString(command, "--github-token ***")
|
||||
|
||||
return command
|
||||
}
|
||||
Reference in New Issue
Block a user