ui/build/upload.go - android_build_soong - Gitiles

 // Copyright 2020 Google Inc. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package build

 // This file contains the functionality to upload data from one location to
 // another.

 import (
 	"bufio"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"

 	"android/soong/shared"
 	"android/soong/ui/metrics"

 	"google.golang.org/protobuf/proto"

 	bazel_metrics_proto "android/soong/ui/metrics/bazel_metrics_proto"
 	upload_proto "android/soong/ui/metrics/upload_proto"
 )

 const (
 	// Used to generate a raw protobuf file that contains information
 	// of the list of metrics files from host to destination storage.
 	uploadPbFilename = ".uploader.pb"
 )

 var (
 	// For testing purpose.
 	tmpDir = ioutil.TempDir
 )

 // pruneMetricsFiles iterates the list of paths, checking if a path exist.
 // If a path is a file, it is added to the return list. If the path is a
 // directory, a recursive call is made to add the children files of the
 // path.
 func pruneMetricsFiles(paths []string) []string {
 	var metricsFiles []string
 	for _, p := range paths {
 		fi, err := os.Stat(p)
 		// Some paths passed may not exist. For example, build errors protobuf
 		// file may not exist since the build was successful.
 		if err != nil {
 			continue
 		}

 		if fi.IsDir() {
 			if l, err := ioutil.ReadDir(p); err != nil {
 				_, _ = fmt.Fprintf(os.Stderr, "Failed to find files under %s\n", p)
 			} else {
 				files := make([]string, 0, len(l))
 				for _, fi := range l {
 					files = append(files, filepath.Join(p, fi.Name()))
 				}
 				metricsFiles = append(metricsFiles, pruneMetricsFiles(files)...)
 			}
 		} else {
 			metricsFiles = append(metricsFiles, p)
 		}
 	}
 	return metricsFiles
 }

 func parseTimingToNanos(str string) int64 {
 	millisString := removeDecimalPoint(str)
 	timingMillis, _ := strconv.ParseInt(millisString, 10, 64)
 	return timingMillis * 1000000
 }

 func parsePercentageToTenThousandths(str string) int32 {
 	percentageString := removeDecimalPoint(str)
 	//remove the % at the end of the string
 	percentage := strings.ReplaceAll(percentageString, "%", "")
 	percentagePortion, _ := strconv.ParseInt(percentage, 10, 32)
 	return int32(percentagePortion)
 }

 func removeDecimalPoint(numString string) string {
 	// The format is always 0.425 or 10.425
 	return strings.ReplaceAll(numString, ".", "")
 }

 func parseTotal(line string) int64 {
 	words := strings.Fields(line)
 	timing := words[3]
 	return parseTimingToNanos(timing)
 }

 func parsePhaseTiming(line string) bazel_metrics_proto.PhaseTiming {
 	words := strings.Fields(line)
 	getPhaseNameAndTimingAndPercentage := func([]string) (string, int64, int32) {
 		// Sample lines include:
 		// Total launch phase time   0.011 s    2.59%
 		// Total target pattern evaluation phase time  0.011 s    2.59%
 		var beginning int
 		var end int
 		for ind, word := range words {
 			if word == "Total" {
 				beginning = ind + 1
 			} else if beginning > 0 && word == "phase" {
 				end = ind
 				break
 			}
 		}
 		phaseName := strings.Join(words[beginning:end], " ")

 		// end is now "phase" - advance by 2 for timing and 4 for percentage
 		percentageString := words[end+4]
 		timingString := words[end+2]
 		timing := parseTimingToNanos(timingString)
 		percentagePortion := parsePercentageToTenThousandths(percentageString)
 		return phaseName, timing, percentagePortion
 	}

 	phaseName, timing, portion := getPhaseNameAndTimingAndPercentage(words)
 	phaseTiming := bazel_metrics_proto.PhaseTiming{}
 	phaseTiming.DurationNanos = &timing
 	phaseTiming.PortionOfBuildTime = &portion

 	phaseTiming.PhaseName = &phaseName
 	return phaseTiming
 }

 // This method takes a file created by bazel's --analyze-profile mode and
 // writes bazel metrics data to the provided filepath.
 // TODO(b/279987768) - move this outside of upload.go
 func processBazelMetrics(bazelProfileFile string, bazelMetricsFile string, ctx Context) {
 	if bazelProfileFile == "" {
 		return
 	}

 	readBazelProto := func(filepath string) bazel_metrics_proto.BazelMetrics {
 		//serialize the proto, write it
 		bazelMetrics := bazel_metrics_proto.BazelMetrics{}

 		file, err := os.ReadFile(filepath)
 		if err != nil {
 			ctx.Fatalln("Error reading metrics file\n", err)
 		}

 		scanner := bufio.NewScanner(strings.NewReader(string(file)))
 		scanner.Split(bufio.ScanLines)

 		var phaseTimings []*bazel_metrics_proto.PhaseTiming
 		for scanner.Scan() {
 			line := scanner.Text()
 			if strings.HasPrefix(line, "Total run time") {
 				total := parseTotal(line)
 				bazelMetrics.Total = &total
 			} else if strings.HasPrefix(line, "Total") {
 				phaseTiming := parsePhaseTiming(line)
 				phaseTimings = append(phaseTimings, &phaseTiming)
 			}
 		}
 		bazelMetrics.PhaseTimings = phaseTimings

 		return bazelMetrics
 	}

 	if _, err := os.Stat(bazelProfileFile); err != nil {
 		// We can assume bazel didn't run if the profile doesn't exist
 		return
 	}
 	bazelProto := readBazelProto(bazelProfileFile)
 	shared.Save(&bazelProto, bazelMetricsFile)
 }

 // UploadMetrics uploads a set of metrics files to a server for analysis.
 // The metrics files are first copied to a temporary directory
 // and the uploader is then executed in the background to allow the user/system
 // to continue working. Soong communicates to the uploader through the
 // upload_proto raw protobuf file.
 func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, bazelProfileFile string, bazelMetricsFile string, paths ...string) {
 	ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics")
 	defer ctx.EndTrace()

 	uploader := config.MetricsUploaderApp()
 	processBazelMetrics(bazelProfileFile, bazelMetricsFile, ctx)

 	if uploader == "" {
 		// If the uploader path was not specified, no metrics shall be uploaded.
 		return
 	}

 	// Several of the files might be directories.
 	metricsFiles := pruneMetricsFiles(paths)
 	if len(metricsFiles) == 0 {
 		return
 	}

 	// The temporary directory cannot be deleted as the metrics uploader is started
 	// in the background and requires to exist until the operation is done. The
 	// uploader can delete the directory as it is specified in the upload proto.
 	tmpDir, err := tmpDir("", "upload_metrics")
 	if err != nil {
 		ctx.Fatalf("failed to create a temporary directory to store the list of metrics files: %v\n", err)
 	}

 	for i, src := range metricsFiles {
 		dst := filepath.Join(tmpDir, filepath.Base(src))
 		if _, err := copyFile(src, dst); err != nil {
 			ctx.Fatalf("failed to copy %q to %q: %v\n", src, dst, err)
 		}
 		metricsFiles[i] = dst
 	}

 	// For platform builds, the branch and target name is hardcoded to specific
 	// values for later extraction of the metrics in the data metrics pipeline.
 	data, err := proto.Marshal(&upload_proto.Upload{
 		CreationTimestampMs:   proto.Uint64(uint64(buildStarted.UnixNano() / int64(time.Millisecond))),
 		CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))),
 		BranchName:            proto.String("developer-metrics"),
 		TargetName:            proto.String("platform-build-systems-metrics"),
 		MetricsFiles:          metricsFiles,
 		DirectoriesToDelete:   []string{tmpDir},
 	})
 	if err != nil {
 		ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err)
 	}

 	pbFile := filepath.Join(tmpDir, uploadPbFilename)
 	if err := ioutil.WriteFile(pbFile, data, 0644); err != nil {
 		ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err)
 	}

 	// Start the uploader in the background as it takes several milliseconds to start the uploader
 	// and prepare the metrics for upload. This affects small shell commands like "lunch".
 	cmd := Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile)
 	if simpleOutput {
 		cmd.RunOrFatal()
 	} else {
 		cmd.RunAndStreamOrFatal()
 	}
 }
	// Copyright 2020 Google Inc. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package build

	// This file contains the functionality to upload data from one location to
	// another.

	import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"android/soong/shared"
	"android/soong/ui/metrics"

	"google.golang.org/protobuf/proto"

	bazel_metrics_proto "android/soong/ui/metrics/bazel_metrics_proto"
	upload_proto "android/soong/ui/metrics/upload_proto"
	)

	const (
	// Used to generate a raw protobuf file that contains information
	// of the list of metrics files from host to destination storage.
	uploadPbFilename = ".uploader.pb"
	)

	var (
	// For testing purpose.
	tmpDir = ioutil.TempDir
	)

	// pruneMetricsFiles iterates the list of paths, checking if a path exist.
	// If a path is a file, it is added to the return list. If the path is a
	// directory, a recursive call is made to add the children files of the
	// path.
	func pruneMetricsFiles(paths []string) []string {
	var metricsFiles []string
	for _, p := range paths {
	fi, err := os.Stat(p)
	// Some paths passed may not exist. For example, build errors protobuf
	// file may not exist since the build was successful.
	if err != nil {
	continue
	}

	if fi.IsDir() {
	if l, err := ioutil.ReadDir(p); err != nil {
	_, _ = fmt.Fprintf(os.Stderr, "Failed to find files under %s\n", p)
	} else {
	files := make([]string, 0, len(l))
	for _, fi := range l {
	files = append(files, filepath.Join(p, fi.Name()))
	}
	metricsFiles = append(metricsFiles, pruneMetricsFiles(files)...)
	}
	} else {
	metricsFiles = append(metricsFiles, p)
	}
	}
	return metricsFiles
	}

	func parseTimingToNanos(str string) int64 {
	millisString := removeDecimalPoint(str)
	timingMillis, _ := strconv.ParseInt(millisString, 10, 64)
	return timingMillis * 1000000
	}

	func parsePercentageToTenThousandths(str string) int32 {
	percentageString := removeDecimalPoint(str)
	//remove the % at the end of the string
	percentage := strings.ReplaceAll(percentageString, "%", "")
	percentagePortion, _ := strconv.ParseInt(percentage, 10, 32)
	return int32(percentagePortion)
	}

	func removeDecimalPoint(numString string) string {
	// The format is always 0.425 or 10.425
	return strings.ReplaceAll(numString, ".", "")
	}

	func parseTotal(line string) int64 {
	words := strings.Fields(line)
	timing := words[3]
	return parseTimingToNanos(timing)
	}

	func parsePhaseTiming(line string) bazel_metrics_proto.PhaseTiming {
	words := strings.Fields(line)
	getPhaseNameAndTimingAndPercentage := func([]string) (string, int64, int32) {
	// Sample lines include:
	// Total launch phase time 0.011 s 2.59%
	// Total target pattern evaluation phase time 0.011 s 2.59%
	var beginning int
	var end int
	for ind, word := range words {
	if word == "Total" {
	beginning = ind + 1
	} else if beginning > 0 && word == "phase" {
	end = ind
	break
	}
	}
	phaseName := strings.Join(words[beginning:end], " ")

	// end is now "phase" - advance by 2 for timing and 4 for percentage
	percentageString := words[end+4]
	timingString := words[end+2]
	timing := parseTimingToNanos(timingString)
	percentagePortion := parsePercentageToTenThousandths(percentageString)
	return phaseName, timing, percentagePortion
	}

	phaseName, timing, portion := getPhaseNameAndTimingAndPercentage(words)
	phaseTiming := bazel_metrics_proto.PhaseTiming{}
	phaseTiming.DurationNanos = &timing
	phaseTiming.PortionOfBuildTime = &portion

	phaseTiming.PhaseName = &phaseName
	return phaseTiming
	}

	// This method takes a file created by bazel's --analyze-profile mode and
	// writes bazel metrics data to the provided filepath.
	// TODO(b/279987768) - move this outside of upload.go
	func processBazelMetrics(bazelProfileFile string, bazelMetricsFile string, ctx Context) {
	if bazelProfileFile == "" {
	return
	}

	readBazelProto := func(filepath string) bazel_metrics_proto.BazelMetrics {
	//serialize the proto, write it
	bazelMetrics := bazel_metrics_proto.BazelMetrics{}

	file, err := os.ReadFile(filepath)
	if err != nil {
	ctx.Fatalln("Error reading metrics file\n", err)
	}

	scanner := bufio.NewScanner(strings.NewReader(string(file)))
	scanner.Split(bufio.ScanLines)

	var phaseTimings []*bazel_metrics_proto.PhaseTiming
	for scanner.Scan() {
	line := scanner.Text()
	if strings.HasPrefix(line, "Total run time") {
	total := parseTotal(line)
	bazelMetrics.Total = &total
	} else if strings.HasPrefix(line, "Total") {
	phaseTiming := parsePhaseTiming(line)
	phaseTimings = append(phaseTimings, &phaseTiming)
	}
	}
	bazelMetrics.PhaseTimings = phaseTimings

	return bazelMetrics
	}

	if _, err := os.Stat(bazelProfileFile); err != nil {
	// We can assume bazel didn't run if the profile doesn't exist
	return
	}
	bazelProto := readBazelProto(bazelProfileFile)
	shared.Save(&bazelProto, bazelMetricsFile)
	}

	// UploadMetrics uploads a set of metrics files to a server for analysis.
	// The metrics files are first copied to a temporary directory
	// and the uploader is then executed in the background to allow the user/system
	// to continue working. Soong communicates to the uploader through the
	// upload_proto raw protobuf file.
	func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, bazelProfileFile string, bazelMetricsFile string, paths ...string) {
	ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics")
	defer ctx.EndTrace()

	uploader := config.MetricsUploaderApp()
	processBazelMetrics(bazelProfileFile, bazelMetricsFile, ctx)

	if uploader == "" {
	// If the uploader path was not specified, no metrics shall be uploaded.
	return
	}

	// Several of the files might be directories.
	metricsFiles := pruneMetricsFiles(paths)
	if len(metricsFiles) == 0 {
	return
	}

	// The temporary directory cannot be deleted as the metrics uploader is started
	// in the background and requires to exist until the operation is done. The
	// uploader can delete the directory as it is specified in the upload proto.
	tmpDir, err := tmpDir("", "upload_metrics")
	if err != nil {
	ctx.Fatalf("failed to create a temporary directory to store the list of metrics files: %v\n", err)
	}

	for i, src := range metricsFiles {
	dst := filepath.Join(tmpDir, filepath.Base(src))
	if _, err := copyFile(src, dst); err != nil {
	ctx.Fatalf("failed to copy %q to %q: %v\n", src, dst, err)
	}
	metricsFiles[i] = dst
	}

	// For platform builds, the branch and target name is hardcoded to specific
	// values for later extraction of the metrics in the data metrics pipeline.
	data, err := proto.Marshal(&upload_proto.Upload{
	CreationTimestampMs: proto.Uint64(uint64(buildStarted.UnixNano() / int64(time.Millisecond))),
	CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))),
	BranchName: proto.String("developer-metrics"),
	TargetName: proto.String("platform-build-systems-metrics"),
	MetricsFiles: metricsFiles,
	DirectoriesToDelete: []string{tmpDir},
	})
	if err != nil {
	ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err)
	}

	pbFile := filepath.Join(tmpDir, uploadPbFilename)
	if err := ioutil.WriteFile(pbFile, data, 0644); err != nil {
	ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err)
	}

	// Start the uploader in the background as it takes several milliseconds to start the uploader
	// and prepare the metrics for upload. This affects small shell commands like "lunch".
	cmd := Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile)
	if simpleOutput {
	cmd.RunOrFatal()
	} else {
	cmd.RunAndStreamOrFatal()
	}
	}