测试GPU的占用情况

测试GPU的占用情况

前言

本文简单介绍测试GPU占用情况和模型耗时统计的代码。

一、测试GPU的占用情况

统计脚本

tegrastats --interval 10 > *.txt

import re
import numpy as np
 
 
# 定义正则表达式模式来匹配 GPU、CPU 和显存占用率
gpu_pattern = re.compile(r"GR3D_FREQ (\d+)%@")
cpu_pattern = re.compile(r"CPU \[([0-9%@,]+)\]")
memory_pattern = re.compile(r"RAM (\d+)/(\d+)MB")
 
# 初始化变量来累积 GPU、CPU 和显存的使用率
gpu_usage = []
cpu_usage = [[] for _ in range(12)]  # 假设有 12 个 CPU 核心
memory_usage = []
 
# 读取文件内容
with open('*.txt', 'r') as file:
    for line in file:
        # 提取 GPU 使用率
        gpu_match = gpu_pattern.search(line)
        if gpu_match:
            gpu_usage.append(int(gpu_match.group(1)))
         
        # 提取 CPU 使用率
        cpu_match = cpu_pattern.search(line)
        if cpu_match:
            cpu_data = cpu_match.group(1).split(',')
            for i, cpu in enumerate(cpu_data):
                usage = int(cpu.split('%')[0])
                cpu_usage[i].append(usage)
         
        # 提取显存使用情况
        memory_match = memory_pattern.search(line)
        if memory_match:
            used_memory = int(memory_match.group(1))
            total_memory = int(memory_match.group(2))
            memory_usage.append(used_memory)
 
# 计算平均 GPU 使用率、P99 和最大值
avg_gpu_usage = np.mean(gpu_usage) if gpu_usage else 0
p99_gpu_usage = np.percentile(gpu_usage, 99) if gpu_usage else 0
max_gpu_usage = np.max(gpu_usage) if gpu_usage else 0
 
# 计算每个 CPU 核心的平均使用率、P99 和最大值
avg_cpu_usage = [np.mean(core_usage) if core_usage else 0 for core_usage in cpu_usage]
p99_cpu_usage = [np.percentile(core_usage, 99) if core_usage else 0 for core_usage in cpu_usage]
max_cpu_usage = [np.max(core_usage) if core_usage else 0 for core_usage in cpu_usage]
 
# 计算显存的平均使用率、P99 和最大值
avg_memory_usage = np.mean(memory_usage) if memory_usage else 0
p99_memory_usage = np.percentile(memory_usage, 99) if memory_usage else 0
max_memory_usage = np.max(memory_usage) if memory_usage else 0
 
# 输出结果
print(f"Average GPU Usage: {avg_gpu_usage:.2f}%")
print(f"P99 GPU Usage: {p99_gpu_usage:.2f}%")
print(f"Max GPU Usage: {max_gpu_usage:.2f}%")
 
for i, (avg, p99, max_val) in enumerate(zip(avg_cpu_usage, p99_cpu_usage, max_cpu_usage)):
    print(f"Average CPU Core {i} Usage: {avg:.2f}%")
    print(f"P99 CPU Core {i} Usage: {p99:.2f}%")
    print(f"Max CPU Core {i} Usage: {max_val:.2f}%")
 
print(f"Average Memory Usage: {avg_memory_usage:.2f} MB")
print(f"P99 Memory Usage: {p99_memory_usage:.2f} MB")
print(f"Max Memory Usage: {max_memory_usage:.2f} MB")

二、耗时统计

计算日志中的耗时

import re
import numpy as np

txt_path = '/home/li/11.txt'
# s = '1-2*(60+(-40.35/5.3+1.2)-(-4*3))'
# num = re.findall('\-?\d+\.?\d*',s) 
# print(num)
with open(txt_path, 'r') as f:
    lines = f.readlines()


float_pattern = r'-?\d+\.?\d*e?-?\d*'
float_compile = re.compile(float_pattern)


preprocess_str = 'preprocess time: '
preprocess_times = []


doInference_str = 'doInference time: '
doInference_times = []


postprocess_str = 'postprocess time: '
postprocess_times = []

ai_str = 'ai time: '
ai_times = []

rule_str = 'rule time: '
rule_times = []

all_str = ', timecost ='
all_times = []

count = 0
for line in lines:
    if preprocess_str in line and 'brake' not in line and 'm_nCycleNum' not in line:
        idx = line.index(preprocess_str)
        # print(line, line[idx])
        finded_str = line[idx:]
        # print(finded_str)
        value, *_ = float_compile.findall(finded_str)
        value = float(value)
        if value > 0.05:
            print(preprocess_str, line)#;continue
        preprocess_times.append(value)
    elif doInference_str in line and 'brake' not in line:
        idx = line.index(doInference_str)
        finded_str = line[idx:]
        value, *_ = float_compile.findall(finded_str)
        value = float(value)
        if value > 1.0:
            print(doInference_str, line)
            count += 1
            continue
        doInference_times.append(value)
    elif postprocess_str in line and 'brake' not in line:
        idx = line.index(postprocess_str)
        finded_str = line[idx:]
        value, *_ = float_compile.findall(finded_str)
        value = float(value)
        if value > 0.05:
            print(postprocess_str, line);continue
        postprocess_times.append(value)
    elif ai_str in line and 'brake' not in line:
        idx = line.index(ai_str)
        finded_str = line[idx:]
        value, *_ = float_compile.findall(finded_str)
        value = float(value)

        #if value < 0.01:
        #    continue
        ai_times.append(value)
    elif rule_str in line and 'brake' not in line:
        idx = line.index(rule_str)
        finded_str = line[idx:]
        value, *_ = float_compile.findall(finded_str)
        value = float(value)
        if value > 0.05:
            print(line);continue
        rule_times.append(value)
    elif all_str in line:
        idx = line.index(all_str)
        finded_str = line[idx:]
        value, *_ = float_compile.findall(finded_str)
        value = float(value)
        #if value > 0.05:
        #    print(line);continue
        all_times.append(value)

rule_times = np.array(rule_times)
ai_times = np.array(ai_times)
preprocess_times = np.array(preprocess_times)
doInference_times = np.array(doInference_times)
postprocess_times = np.array(postprocess_times)
all_times = np.array(all_times)
print(f'count={count}')
print(f'total time: {np.mean(all_times)}')
print(f'\trule time: {np.mean(rule_times)}')
print(f'\tai time: {np.mean(ai_times)}')
print(f'\t\tpreprocess time: {np.mean(preprocess_times)}')
print(f'\t\tdoInference time: {np.mean(doInference_times)}')
print(f'\t\tpostprocess time: {np.mean(postprocess_times)}')
print()

for value in range(90, 91):
    print(f'{value}:total_time: {np.percentile(all_times, value, axis=0)}')
    print(f'{value}:rule_time: {np.percentile(rule_times, value, axis=0)}')
    print(f'{value}:ai_time: {np.percentile(ai_times, value, axis=0)}')
    print(f'\t{value}:preprocess_time: {np.percentile(preprocess_times, value, axis=0)}')
    print(f'\t{value}:doInference_time: {np.percentile(doInference_times, value, axis=0)}')
    print(f'\t{value}:postprocess_time: {np.percentile(postprocess_times, value, axis=0)}')
    print()

总结

本文简单介绍测试GPU占用情况和模型耗时统计的代码。