unplugged-system/tools/treble/build/treble_build/local/ninja.go

// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package local

import (
	"bufio"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
	"strings"
	"time"

	"tools/treble/build/report/app"
)

// Performance degrades running multiple CLIs
const (
	MaxNinjaCliWorkers       = 4
	DefaultNinjaTimeout      = "100s"
	DefaultNinjaBuildTimeout = "30m"
)

// Separate out the executable to allow tests to override the results
type ninjaExec interface {
	Command(ctx context.Context, target string) (*bytes.Buffer, error)
	Input(ctx context.Context, target string) (*bytes.Buffer, error)
	Query(ctx context.Context, target string) (*bytes.Buffer, error)
	Path(ctx context.Context, target string, dependency string) (*bytes.Buffer, error)
	Paths(ctx context.Context, target string, dependency string) (*bytes.Buffer, error)
	Deps(ctx context.Context) (*bytes.Buffer, error)
	Build(ctx context.Context, target string) (*bytes.Buffer, error)
}

// Parse data

// Add all lines to a given array removing any leading whitespace
func linesToArray(s *bufio.Scanner, arr *[]string) {
	for s.Scan() {
		line := strings.TrimSpace(s.Text())
		*arr = append(*arr, line)
	}
}

// parse -t commands
func parseCommand(target string, data *bytes.Buffer) (*app.BuildCommand, error) {
	out := &app.BuildCommand{Target: target, Cmds: []string{}}
	s := bufio.NewScanner(data)
	// This tool returns all the commands needed to build a target.
	// When running against a target like droid the default capacity
	// will be overrun.   Extend the capacity here.
	const capacity = 1024 * 1024
	buf := make([]byte, capacity)
	s.Buffer(buf, capacity)
	linesToArray(s, &out.Cmds)
	return out, nil
}

// parse -t inputs
func parseInput(target string, data *bytes.Buffer) (*app.BuildInput, error) {
	out := &app.BuildInput{Target: target, Files: []string{}}
	s := bufio.NewScanner(data)
	linesToArray(s, &out.Files)
	return out, nil
}

// parse -t query
func parseQuery(target string, data *bytes.Buffer) (*app.BuildQuery, error) {
	out := &app.BuildQuery{Target: target, Inputs: []string{}, Outputs: []string{}}
	const (
		unknown = iota
		inputs
		outputs
	)
	state := unknown
	s := bufio.NewScanner(data)
	for s.Scan() {
		line := strings.TrimSpace(s.Text())
		if strings.HasPrefix(line, "input:") {
			state = inputs
		} else if strings.HasPrefix(line, "outputs:") {
			state = outputs
		} else {
			switch state {
			case inputs:
				out.Inputs = append(out.Inputs, line)
			case outputs:
				out.Outputs = append(out.Outputs, line)
			}
		}
	}
	return out, nil
}

// parse -t path
func parsePath(target string, dependency string, data *bytes.Buffer) (*app.BuildPath, error) {
	out := &app.BuildPath{Target: target, Dependency: dependency, Paths: []string{}}
	s := bufio.NewScanner(data)
	linesToArray(s, &out.Paths)
	return out, nil
}

// parse -t paths
func parsePaths(target string, dependency string, data *bytes.Buffer) ([]*app.BuildPath, error) {
	out := []*app.BuildPath{}
	s := bufio.NewScanner(data)
	for s.Scan() {
		path := strings.Fields(s.Text())
		out = append(out, &app.BuildPath{Target: target, Dependency: dependency, Paths: path})
	}
	return out, nil
}

// parse build output
func parseBuild(target string, data *bytes.Buffer, success bool) *app.BuildCmdResult {
	out := &app.BuildCmdResult{Name: target, Output: []string{}}
	s := bufio.NewScanner(data)
	out.Success = success
	linesToArray(s, &out.Output)
	return out
}

// parse deps command
func parseDeps(data *bytes.Buffer) (*app.BuildDeps, error) {
	out := &app.BuildDeps{Targets: make(map[string][]string)}
	s := bufio.NewScanner(data)
	curTarget := ""
	var deps []string
	for s.Scan() {
		line := strings.TrimSpace(s.Text())
		// Check if it's a new target
		tokens := strings.Split(line, ":")
		if len(tokens) > 1 {
			if curTarget != "" {
				out.Targets[curTarget] = deps
			}
			deps = []string{}
			curTarget = tokens[0]
		} else if line != "" {
			deps = append(deps, line)
		}

	}
	if curTarget != "" {
		out.Targets[curTarget] = deps
	}
	return out, nil
}

//
// Command line interface to ninja binary.
//
// This file implements the ninja.Ninja interface by querying
// the build graph via the ninja binary.  The mapping between
// the interface and the binary are as follows:
//    Command()   -t commands
//    Input()     -t inputs
//    Query()     -t query
//    Path()      -t path
//    Paths()     -t paths
//    Deps()      -t deps
//
//

type ninjaCmd struct {
	cmd string
	db  string

	clientMode   bool
	timeout      time.Duration
	buildTimeout time.Duration
}

func (n *ninjaCmd) runTool(ctx context.Context, tool string, targets []string) (out *bytes.Buffer, err error) {

	args := []string{"-f", n.db}

	if n.clientMode {
		args = append(args, []string{
			"-t", "client",
			"-c", tool}...)
	} else {
		args = append(args, []string{"-t", tool}...)
	}
	args = append(args, targets...)
	data := []byte{}
	err, _ = runPipe(ctx, n.timeout, n.cmd, args, func(r io.Reader) {
		data, _ = ioutil.ReadAll(r)
	})
	return bytes.NewBuffer(data), err

}
func (n *ninjaCmd) Command(ctx context.Context, target string) (*bytes.Buffer, error) {
	return n.runTool(ctx, "commands", []string{target})
}
func (n *ninjaCmd) Input(ctx context.Context, target string) (*bytes.Buffer, error) {
	return n.runTool(ctx, "inputs", []string{target})
}
func (n *ninjaCmd) Query(ctx context.Context, target string) (*bytes.Buffer, error) {
	return n.runTool(ctx, "query", []string{target})
}
func (n *ninjaCmd) Path(ctx context.Context, target string, dependency string) (*bytes.Buffer, error) {
	return n.runTool(ctx, "path", []string{target, dependency})
}
func (n *ninjaCmd) Paths(ctx context.Context, target string, dependency string) (*bytes.Buffer, error) {
	return n.runTool(ctx, "paths", []string{target, dependency})
}
func (n *ninjaCmd) Deps(ctx context.Context) (*bytes.Buffer, error) {
	return n.runTool(ctx, "deps", []string{})
}

func (n *ninjaCmd) Build(ctx context.Context, target string) (*bytes.Buffer, error) {

	args := append([]string{
		"-f", n.db,
		target})
	data := []byte{}
	err, _ := runPipe(ctx, n.buildTimeout, n.cmd, args, func(r io.Reader) {
		data, _ = ioutil.ReadAll(r)
	})

	return bytes.NewBuffer(data), err
}

// Command line ninja
type ninjaCli struct {
	n ninjaExec
}

// ninja -t commands
func (cli *ninjaCli) Command(ctx context.Context, target string) (*app.BuildCommand, error) {
	raw, err := cli.n.Command(ctx, target)
	if err != nil {
		return nil, err
	}
	return parseCommand(target, raw)
}

// ninja -t inputs
func (cli *ninjaCli) Input(ctx context.Context, target string) (*app.BuildInput, error) {
	raw, err := cli.n.Input(ctx, target)
	if err != nil {
		return nil, err
	}
	return parseInput(target, raw)
}

// ninja -t query
func (cli *ninjaCli) Query(ctx context.Context, target string) (*app.BuildQuery, error) {
	raw, err := cli.n.Query(ctx, target)
	if err != nil {
		return nil, err
	}
	return parseQuery(target, raw)
}

// ninja -t path
func (cli *ninjaCli) Path(ctx context.Context, target string, dependency string) (*app.BuildPath, error) {
	raw, err := cli.n.Path(ctx, target, dependency)
	if err != nil {
		return nil, err
	}
	return parsePath(target, dependency, raw)
}

// ninja -t paths
func (cli *ninjaCli) Paths(ctx context.Context, target string, dependency string) ([]*app.BuildPath, error) {
	raw, err := cli.n.Paths(ctx, target, dependency)
	if err != nil {
		return nil, err
	}
	return parsePaths(target, dependency, raw)
}

// ninja -t deps
func (cli *ninjaCli) Deps(ctx context.Context) (*app.BuildDeps, error) {
	raw, err := cli.n.Deps(ctx)
	if err != nil {
		return nil, err
	}
	return parseDeps(raw)
}

// Build given target
func (cli *ninjaCli) Build(ctx context.Context, target string) *app.BuildCmdResult {
	raw, err := cli.n.Build(ctx, target)
	return parseBuild(target, raw, err == nil)

}

// Wait for server
func (cli *ninjaCli) WaitForServer(ctx context.Context, maxTries int) error {
	// Wait for server to response to an empty input request
	fmt.Printf("Waiting for server.")
	for i := 0; i < maxTries; i++ {
		_, err := cli.Input(ctx, "")
		if err == nil {
			fmt.Printf("\nConnected\n")
			return nil
		}
		fmt.Printf(".")
		time.Sleep(time.Second)
	}
	fmt.Printf(" failed\n")
	return errors.New("Failed to connect")
}
func NewNinjaCli(cmd string, db string, timeout, buildTimeout time.Duration, client bool) *ninjaCli {
	cli := &ninjaCli{n: &ninjaCmd{cmd: cmd, db: db, timeout: timeout, buildTimeout: buildTimeout, clientMode: client}}
	return cli
}

type ninjaServer struct {
	cmdName string
	db      string
	ctx     *exec.Cmd
}

// Run server
func (srv *ninjaServer) Start(ctx context.Context) error {
	args := []string{"-f", srv.db, "-t", "server"}
	srv.ctx = exec.CommandContext(ctx, srv.cmdName, args[0:]...)
	err := srv.ctx.Start()
	if err != nil {
		return err
	}
	srv.ctx.Wait()
	return nil
}
func (srv *ninjaServer) Kill() {
	if srv.ctx != nil {
		srv.ctx.Process.Kill()
	}
}
func NewNinjaServer(cmd string, db string) *ninjaServer {
	return &ninjaServer{cmdName: cmd, db: db}
}