From: Nicolai Hähnle <***@amd.com>
---
src/gallium/drivers/radeon/r600_perfcounter.c | 639 ----------------
src/gallium/drivers/radeonsi/Makefile.sources | 1 -
src/gallium/drivers/radeonsi/meson.build | 1 -
src/gallium/drivers/radeonsi/si_perfcounter.c | 688 ++++++++++++++++--
src/gallium/drivers/radeonsi/si_pipe.c | 2 +-
src/gallium/drivers/radeonsi/si_pipe.h | 4 +-
src/gallium/drivers/radeonsi/si_query.h | 74 +-
7 files changed, 643 insertions(+), 766 deletions(-)
delete mode 100644 src/gallium/drivers/radeon/r600_perfcounter.c
diff --git a/src/gallium/drivers/radeon/r600_perfcounter.c b/src/gallium/drivers/radeon/r600_perfcounter.c
deleted file mode 100644
index 57c3246898a..00000000000
--- a/src/gallium/drivers/radeon/r600_perfcounter.c
+++ /dev/null
@@ -1,639 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "util/u_memory.h"
-#include "radeonsi/si_query.h"
-#include "radeonsi/si_pipe.h"
-#include "amd/common/sid.h"
-
-/* Max counters per HW block */
-#define SI_QUERY_MAX_COUNTERS 16
-
-static struct si_perfcounter_block *
-lookup_counter(struct si_perfcounters *pc, unsigned index,
- unsigned *base_gid, unsigned *sub_index)
-{
- struct si_perfcounter_block *block = pc->blocks;
- unsigned bid;
-
- *base_gid = 0;
- for (bid = 0; bid < pc->num_blocks; ++bid, ++block) {
- unsigned total = block->num_groups * block->num_selectors;
-
- if (index < total) {
- *sub_index = index;
- return block;
- }
-
- index -= total;
- *base_gid += block->num_groups;
- }
-
- return NULL;
-}
-
-static struct si_perfcounter_block *
-lookup_group(struct si_perfcounters *pc, unsigned *index)
-{
- unsigned bid;
- struct si_perfcounter_block *block = pc->blocks;
-
- for (bid = 0; bid < pc->num_blocks; ++bid, ++block) {
- if (*index < block->num_groups)
- return block;
- *index -= block->num_groups;
- }
-
- return NULL;
-}
-
-struct si_pc_group {
- struct si_pc_group *next;
- struct si_perfcounter_block *block;
- unsigned sub_gid; /* only used during init */
- unsigned result_base; /* only used during init */
- int se;
- int instance;
- unsigned num_counters;
- unsigned selectors[SI_QUERY_MAX_COUNTERS];
-};
-
-struct si_pc_counter {
- unsigned base;
- unsigned qwords;
- unsigned stride; /* in uint64s */
-};
-
-#define SI_PC_SHADERS_WINDOWING (1 << 31)
-
-struct si_query_pc {
- struct si_query_hw b;
-
- unsigned shaders;
- unsigned num_counters;
- struct si_pc_counter *counters;
- struct si_pc_group *groups;
-};
-
-static void si_pc_query_destroy(struct si_screen *sscreen,
- struct si_query *rquery)
-{
- struct si_query_pc *query = (struct si_query_pc *)rquery;
-
- while (query->groups) {
- struct si_pc_group *group = query->groups;
- query->groups = group->next;
- FREE(group);
- }
-
- FREE(query->counters);
-
- si_query_hw_destroy(sscreen, rquery);
-}
-
-static bool si_pc_query_prepare_buffer(struct si_screen *screen,
- struct si_query_hw *hwquery,
- struct r600_resource *buffer)
-{
- /* no-op */
- return true;
-}
-
-static void si_pc_query_emit_start(struct si_context *sctx,
- struct si_query_hw *hwquery,
- struct r600_resource *buffer, uint64_t va)
-{
- struct si_perfcounters *pc = sctx->screen->perfcounters;
- struct si_query_pc *query = (struct si_query_pc *)hwquery;
- struct si_pc_group *group;
- int current_se = -1;
- int current_instance = -1;
-
- if (query->shaders)
- pc->emit_shaders(sctx, query->shaders);
-
- for (group = query->groups; group; group = group->next) {
- struct si_perfcounter_block *block = group->block;
-
- if (group->se != current_se || group->instance != current_instance) {
- current_se = group->se;
- current_instance = group->instance;
- pc->emit_instance(sctx, group->se, group->instance);
- }
-
- pc->emit_select(sctx, block, group->num_counters, group->selectors);
- }
-
- if (current_se != -1 || current_instance != -1)
- pc->emit_instance(sctx, -1, -1);
-
- pc->emit_start(sctx, buffer, va);
-}
-
-static void si_pc_query_emit_stop(struct si_context *sctx,
- struct si_query_hw *hwquery,
- struct r600_resource *buffer, uint64_t va)
-{
- struct si_perfcounters *pc = sctx->screen->perfcounters;
- struct si_query_pc *query = (struct si_query_pc *)hwquery;
- struct si_pc_group *group;
-
- pc->emit_stop(sctx, buffer, va);
-
- for (group = query->groups; group; group = group->next) {
- struct si_perfcounter_block *block = group->block;
- unsigned se = group->se >= 0 ? group->se : 0;
- unsigned se_end = se + 1;
-
- if ((block->flags & SI_PC_BLOCK_SE) && (group->se < 0))
- se_end = sctx->screen->info.max_se;
-
- do {
- unsigned instance = group->instance >= 0 ? group->instance : 0;
-
- do {
- pc->emit_instance(sctx, se, instance);
- pc->emit_read(sctx, block,
- group->num_counters, group->selectors,
- buffer, va);
- va += sizeof(uint64_t) * group->num_counters;
- } while (group->instance < 0 && ++instance < block->num_instances);
- } while (++se < se_end);
- }
-
- pc->emit_instance(sctx, -1, -1);
-}
-
-static void si_pc_query_clear_result(struct si_query_hw *hwquery,
- union pipe_query_result *result)
-{
- struct si_query_pc *query = (struct si_query_pc *)hwquery;
-
- memset(result, 0, sizeof(result->batch[0]) * query->num_counters);
-}
-
-static void si_pc_query_add_result(struct si_screen *sscreen,
- struct si_query_hw *hwquery,
- void *buffer,
- union pipe_query_result *result)
-{
- struct si_query_pc *query = (struct si_query_pc *)hwquery;
- uint64_t *results = buffer;
- unsigned i, j;
-
- for (i = 0; i < query->num_counters; ++i) {
- struct si_pc_counter *counter = &query->counters[i];
-
- for (j = 0; j < counter->qwords; ++j) {
- uint32_t value = results[counter->base + j * counter->stride];
- result->batch[i].u64 += value;
- }
- }
-}
-
-static struct si_query_ops batch_query_ops = {
- .destroy = si_pc_query_destroy,
- .begin = si_query_hw_begin,
- .end = si_query_hw_end,
- .get_result = si_query_hw_get_result
-};
-
-static struct si_query_hw_ops batch_query_hw_ops = {
- .prepare_buffer = si_pc_query_prepare_buffer,
- .emit_start = si_pc_query_emit_start,
- .emit_stop = si_pc_query_emit_stop,
- .clear_result = si_pc_query_clear_result,
- .add_result = si_pc_query_add_result,
-};
-
-static struct si_pc_group *get_group_state(struct si_screen *screen,
- struct si_query_pc *query,
- struct si_perfcounter_block *block,
- unsigned sub_gid)
-{
- struct si_pc_group *group = query->groups;
-
- while (group) {
- if (group->block == block && group->sub_gid == sub_gid)
- return group;
- group = group->next;
- }
-
- group = CALLOC_STRUCT(si_pc_group);
- if (!group)
- return NULL;
-
- group->block = block;
- group->sub_gid = sub_gid;
-
- if (block->flags & SI_PC_BLOCK_SHADER) {
- unsigned sub_gids = block->num_instances;
- unsigned shader_id;
- unsigned shaders;
- unsigned query_shaders;
-
- if (block->flags & SI_PC_BLOCK_SE_GROUPS)
- sub_gids = sub_gids * screen->info.max_se;
- shader_id = sub_gid / sub_gids;
- sub_gid = sub_gid % sub_gids;
-
- shaders = screen->perfcounters->shader_type_bits[shader_id];
-
- query_shaders = query->shaders & ~SI_PC_SHADERS_WINDOWING;
- if (query_shaders && query_shaders != shaders) {
- fprintf(stderr, "si_perfcounter: incompatible shader groups\n");
- FREE(group);
- return NULL;
- }
- query->shaders = shaders;
- }
-
- if (block->flags & SI_PC_BLOCK_SHADER_WINDOWED && !query->shaders) {
- // A non-zero value in query->shaders ensures that the shader
- // masking is reset unless the user explicitly requests one.
- query->shaders = SI_PC_SHADERS_WINDOWING;
- }
-
- if (block->flags & SI_PC_BLOCK_SE_GROUPS) {
- group->se = sub_gid / block->num_instances;
- sub_gid = sub_gid % block->num_instances;
- } else {
- group->se = -1;
- }
-
- if (block->flags & SI_PC_BLOCK_INSTANCE_GROUPS) {
- group->instance = sub_gid;
- } else {
- group->instance = -1;
- }
-
- group->next = query->groups;
- query->groups = group;
-
- return group;
-}
-
-struct pipe_query *si_create_batch_query(struct pipe_context *ctx,
- unsigned num_queries,
- unsigned *query_types)
-{
- struct si_screen *screen =
- (struct si_screen *)ctx->screen;
- struct si_perfcounters *pc = screen->perfcounters;
- struct si_perfcounter_block *block;
- struct si_pc_group *group;
- struct si_query_pc *query;
- unsigned base_gid, sub_gid, sub_index;
- unsigned i, j;
-
- if (!pc)
- return NULL;
-
- query = CALLOC_STRUCT(si_query_pc);
- if (!query)
- return NULL;
-
- query->b.b.ops = &batch_query_ops;
- query->b.ops = &batch_query_hw_ops;
-
- query->num_counters = num_queries;
-
- /* Collect selectors per group */
- for (i = 0; i < num_queries; ++i) {
- unsigned sub_gid;
-
- if (query_types[i] < SI_QUERY_FIRST_PERFCOUNTER)
- goto error;
-
- block = lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER,
- &base_gid, &sub_index);
- if (!block)
- goto error;
-
- sub_gid = sub_index / block->num_selectors;
- sub_index = sub_index % block->num_selectors;
-
- group = get_group_state(screen, query, block, sub_gid);
- if (!group)
- goto error;
-
- if (group->num_counters >= block->num_counters) {
- fprintf(stderr,
- "perfcounter group %s: too many selected\n",
- block->basename);
- goto error;
- }
- group->selectors[group->num_counters] = sub_index;
- ++group->num_counters;
- }
-
- /* Compute result bases and CS size per group */
- query->b.num_cs_dw_end = pc->num_stop_cs_dwords;
- query->b.num_cs_dw_end += pc->num_instance_cs_dwords;
-
- i = 0;
- for (group = query->groups; group; group = group->next) {
- struct si_perfcounter_block *block = group->block;
- unsigned read_dw;
- unsigned instances = 1;
-
- if ((block->flags & SI_PC_BLOCK_SE) && group->se < 0)
- instances = screen->info.max_se;
- if (group->instance < 0)
- instances *= block->num_instances;
-
- group->result_base = i;
- query->b.result_size += sizeof(uint64_t) * instances * group->num_counters;
- i += instances * group->num_counters;
-
- read_dw = 6 * group->num_counters;
- query->b.num_cs_dw_end += instances * read_dw;
- query->b.num_cs_dw_end += instances * pc->num_instance_cs_dwords;
- }
-
- if (query->shaders) {
- if (query->shaders == SI_PC_SHADERS_WINDOWING)
- query->shaders = 0xffffffff;
- }
-
- /* Map user-supplied query array to result indices */
- query->counters = CALLOC(num_queries, sizeof(*query->counters));
- for (i = 0; i < num_queries; ++i) {
- struct si_pc_counter *counter = &query->counters[i];
- struct si_perfcounter_block *block;
-
- block = lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER,
- &base_gid, &sub_index);
-
- sub_gid = sub_index / block->num_selectors;
- sub_index = sub_index % block->num_selectors;
-
- group = get_group_state(screen, query, block, sub_gid);
- assert(group != NULL);
-
- for (j = 0; j < group->num_counters; ++j) {
- if (group->selectors[j] == sub_index)
- break;
- }
-
- counter->base = group->result_base + j;
- counter->stride = group->num_counters;
-
- counter->qwords = 1;
- if ((block->flags & SI_PC_BLOCK_SE) && group->se < 0)
- counter->qwords = screen->info.max_se;
- if (group->instance < 0)
- counter->qwords *= block->num_instances;
- }
-
- if (!si_query_hw_init(screen, &query->b))
- goto error;
-
- return (struct pipe_query *)query;
-
-error:
- si_pc_query_destroy(screen, &query->b.b);
- return NULL;
-}
-
-static bool si_init_block_names(struct si_screen *screen,
- struct si_perfcounter_block *block)
-{
- unsigned i, j, k;
- unsigned groups_shader = 1, groups_se = 1, groups_instance = 1;
- unsigned namelen;
- char *groupname;
- char *p;
-
- if (block->flags & SI_PC_BLOCK_INSTANCE_GROUPS)
- groups_instance = block->num_instances;
- if (block->flags & SI_PC_BLOCK_SE_GROUPS)
- groups_se = screen->info.max_se;
- if (block->flags & SI_PC_BLOCK_SHADER)
- groups_shader = screen->perfcounters->num_shader_types;
-
- namelen = strlen(block->basename);
- block->group_name_stride = namelen + 1;
- if (block->flags & SI_PC_BLOCK_SHADER)
- block->group_name_stride += 3;
- if (block->flags & SI_PC_BLOCK_SE_GROUPS) {
- assert(groups_se <= 10);
- block->group_name_stride += 1;
-
- if (block->flags & SI_PC_BLOCK_INSTANCE_GROUPS)
- block->group_name_stride += 1;
- }
- if (block->flags & SI_PC_BLOCK_INSTANCE_GROUPS) {
- assert(groups_instance <= 100);
- block->group_name_stride += 2;
- }
-
- block->group_names = MALLOC(block->num_groups * block->group_name_stride);
- if (!block->group_names)
- return false;
-
- groupname = block->group_names;
- for (i = 0; i < groups_shader; ++i) {
- const char *shader_suffix = screen->perfcounters->shader_type_suffixes[i];
- unsigned shaderlen = strlen(shader_suffix);
- for (j = 0; j < groups_se; ++j) {
- for (k = 0; k < groups_instance; ++k) {
- strcpy(groupname, block->basename);
- p = groupname + namelen;
-
- if (block->flags & SI_PC_BLOCK_SHADER) {
- strcpy(p, shader_suffix);
- p += shaderlen;
- }
-
- if (block->flags & SI_PC_BLOCK_SE_GROUPS) {
- p += sprintf(p, "%d", j);
- if (block->flags & SI_PC_BLOCK_INSTANCE_GROUPS)
- *p++ = '_';
- }
-
- if (block->flags & SI_PC_BLOCK_INSTANCE_GROUPS)
- p += sprintf(p, "%d", k);
-
- groupname += block->group_name_stride;
- }
- }
- }
-
- assert(block->num_selectors <= 1000);
- block->selector_name_stride = block->group_name_stride + 4;
- block->selector_names = MALLOC(block->num_groups * block->num_selectors *
- block->selector_name_stride);
- if (!block->selector_names)
- return false;
-
- groupname = block->group_names;
- p = block->selector_names;
- for (i = 0; i < block->num_groups; ++i) {
- for (j = 0; j < block->num_selectors; ++j) {
- sprintf(p, "%s_%03d", groupname, j);
- p += block->selector_name_stride;
- }
- groupname += block->group_name_stride;
- }
-
- return true;
-}
-
-int si_get_perfcounter_info(struct si_screen *screen,
- unsigned index,
- struct pipe_driver_query_info *info)
-{
- struct si_perfcounters *pc = screen->perfcounters;
- struct si_perfcounter_block *block;
- unsigned base_gid, sub;
-
- if (!pc)
- return 0;
-
- if (!info) {
- unsigned bid, num_queries = 0;
-
- for (bid = 0; bid < pc->num_blocks; ++bid) {
- num_queries += pc->blocks[bid].num_selectors *
- pc->blocks[bid].num_groups;
- }
-
- return num_queries;
- }
-
- block = lookup_counter(pc, index, &base_gid, &sub);
- if (!block)
- return 0;
-
- if (!block->selector_names) {
- if (!si_init_block_names(screen, block))
- return 0;
- }
- info->name = block->selector_names + sub * block->selector_name_stride;
- info->query_type = SI_QUERY_FIRST_PERFCOUNTER + index;
- info->max_value.u64 = 0;
- info->type = PIPE_DRIVER_QUERY_TYPE_UINT64;
- info->result_type = PIPE_DRIVER_QUERY_RESULT_TYPE_AVERAGE;
- info->group_id = base_gid + sub / block->num_selectors;
- info->flags = PIPE_DRIVER_QUERY_FLAG_BATCH;
- if (sub > 0 && sub + 1 < block->num_selectors * block->num_groups)
- info->flags |= PIPE_DRIVER_QUERY_FLAG_DONT_LIST;
- return 1;
-}
-
-int si_get_perfcounter_group_info(struct si_screen *screen,
- unsigned index,
- struct pipe_driver_query_group_info *info)
-{
- struct si_perfcounters *pc = screen->perfcounters;
- struct si_perfcounter_block *block;
-
- if (!pc)
- return 0;
-
- if (!info)
- return pc->num_groups;
-
- block = lookup_group(pc, &index);
- if (!block)
- return 0;
-
- if (!block->group_names) {
- if (!si_init_block_names(screen, block))
- return 0;
- }
- info->name = block->group_names + index * block->group_name_stride;
- info->num_queries = block->num_selectors;
- info->max_active_queries = block->num_counters;
- return 1;
-}
-
-void si_perfcounters_destroy(struct si_screen *sscreen)
-{
- if (sscreen->perfcounters)
- sscreen->perfcounters->cleanup(sscreen);
-}
-
-bool si_perfcounters_init(struct si_perfcounters *pc,
- unsigned num_blocks)
-{
- pc->blocks = CALLOC(num_blocks, sizeof(struct si_perfcounter_block));
- if (!pc->blocks)
- return false;
-
- pc->separate_se = debug_get_bool_option("RADEON_PC_SEPARATE_SE", false);
- pc->separate_instance = debug_get_bool_option("RADEON_PC_SEPARATE_INSTANCE", false);
-
- return true;
-}
-
-void si_perfcounters_add_block(struct si_screen *sscreen,
- struct si_perfcounters *pc,
- const char *name, unsigned flags,
- unsigned counters, unsigned selectors,
- unsigned instances, void *data)
-{
- struct si_perfcounter_block *block = &pc->blocks[pc->num_blocks];
-
- assert(counters <= SI_QUERY_MAX_COUNTERS);
-
- block->basename = name;
- block->flags = flags;
- block->num_counters = counters;
- block->num_selectors = selectors;
- block->num_instances = MAX2(instances, 1);
- block->data = data;
-
- if (pc->separate_se && (block->flags & SI_PC_BLOCK_SE))
- block->flags |= SI_PC_BLOCK_SE_GROUPS;
- if (pc->separate_instance && block->num_instances > 1)
- block->flags |= SI_PC_BLOCK_INSTANCE_GROUPS;
-
- if (block->flags & SI_PC_BLOCK_INSTANCE_GROUPS) {
- block->num_groups = block->num_instances;
- } else {
- block->num_groups = 1;
- }
-
- if (block->flags & SI_PC_BLOCK_SE_GROUPS)
- block->num_groups *= sscreen->info.max_se;
- if (block->flags & SI_PC_BLOCK_SHADER)
- block->num_groups *= pc->num_shader_types;
-
- ++pc->num_blocks;
- pc->num_groups += block->num_groups;
-}
-
-void si_perfcounters_do_destroy(struct si_perfcounters *pc)
-{
- unsigned i;
-
- for (i = 0; i < pc->num_blocks; ++i) {
- FREE(pc->blocks[i].group_names);
- FREE(pc->blocks[i].selector_names);
- }
- FREE(pc->blocks);
- FREE(pc);
-}
diff --git a/src/gallium/drivers/radeonsi/Makefile.sources b/src/gallium/drivers/radeonsi/Makefile.sources
index dcaf446effe..713629c6e87 100644
--- a/src/gallium/drivers/radeonsi/Makefile.sources
+++ b/src/gallium/drivers/radeonsi/Makefile.sources
@@ -42,21 +42,20 @@ C_SOURCES := \
si_state_draw.c \
si_state_msaa.c \
si_state_shaders.c \
si_state_streamout.c \
si_state_viewport.c \
si_state.h \
si_test_dma.c \
si_test_dma_perf.c \
si_texture.c \
si_uvd.c \
- ../radeon/r600_perfcounter.c \
../radeon/radeon_uvd.c \
../radeon/radeon_uvd.h \
../radeon/radeon_vcn_dec_jpeg.c \
../radeon/radeon_vcn_dec.c \
../radeon/radeon_vcn_dec.h \
../radeon/radeon_vcn_enc_1_2.c \
../radeon/radeon_vcn_enc.c \
../radeon/radeon_vcn_enc.h \
../radeon/radeon_uvd_enc_1_1.c \
../radeon/radeon_uvd_enc.c \
diff --git a/src/gallium/drivers/radeonsi/meson.build b/src/gallium/drivers/radeonsi/meson.build
index ac8ed949ea0..cf3b24cd358 100644
--- a/src/gallium/drivers/radeonsi/meson.build
+++ b/src/gallium/drivers/radeonsi/meson.build
@@ -58,21 +58,20 @@ files_libradeonsi = files(
'si_state_binning.c',
'si_state_draw.c',
'si_state_msaa.c',
'si_state_shaders.c',
'si_state_streamout.c',
'si_state_viewport.c',
'si_test_dma.c',
'si_test_dma_perf.c',
'si_texture.c',
'si_uvd.c',
- '../radeon/r600_perfcounter.c',
'../radeon/radeon_uvd.c',
'../radeon/radeon_uvd.h',
'../radeon/radeon_vcn_enc_1_2.c',
'../radeon/radeon_vcn_enc.c',
'../radeon/radeon_vcn_enc.h',
'../radeon/radeon_vcn_dec_jpeg.c',
'../radeon/radeon_vcn_dec.c',
'../radeon/radeon_vcn_dec.h',
'../radeon/radeon_uvd_enc_1_1.c',
'../radeon/radeon_uvd_enc.c',
diff --git a/src/gallium/drivers/radeonsi/si_perfcounter.c b/src/gallium/drivers/radeonsi/si_perfcounter.c
index cea7d57e518..69e149c76b6 100644
--- a/src/gallium/drivers/radeonsi/si_perfcounter.c
+++ b/src/gallium/drivers/radeonsi/si_perfcounter.c
@@ -20,20 +20,38 @@
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "si_build_pm4.h"
#include "si_query.h"
#include "util/u_memory.h"
+enum si_pc_block_flags {
+ /* This block is part of the shader engine */
+ SI_PC_BLOCK_SE = (1 << 0),
+
+ /* Expose per-instance groups instead of summing all instances (within
+ * an SE). */
+ SI_PC_BLOCK_INSTANCE_GROUPS = (1 << 1),
+
+ /* Expose per-SE groups instead of summing instances across SEs. */
+ SI_PC_BLOCK_SE_GROUPS = (1 << 2),
+
+ /* Shader block */
+ SI_PC_BLOCK_SHADER = (1 << 3),
+
+ /* Non-shader block with perfcounters windowed by shaders. */
+ SI_PC_BLOCK_SHADER_WINDOWED = (1 << 4),
+};
+
enum si_pc_reg_layout {
/* All secondary selector dwords follow as one block after the primary
* selector dwords for the counters that have secondary selectors.
*/
SI_PC_MULTI_BLOCK = 0,
/* Each secondary selector dword follows immediately afters the
* corresponding primary.
*/
SI_PC_MULTI_ALTERNATE = 1,
@@ -62,44 +80,88 @@ struct si_pc_block_base {
unsigned select_or;
unsigned select0;
unsigned counter0_lo;
unsigned *select;
unsigned *counters;
unsigned num_multi;
unsigned num_prelude;
unsigned layout;
};
-struct si_pc_block {
+struct si_pc_block_gfxdescr {
struct si_pc_block_base *b;
unsigned selectors;
unsigned instances;
};
+struct si_pc_block {
+ const struct si_pc_block_gfxdescr *b;
+ unsigned num_instances;
+
+ unsigned num_groups;
+ char *group_names;
+ unsigned group_name_stride;
+
+ char *selector_names;
+ unsigned selector_name_stride;
+};
+
/* The order is chosen to be compatible with GPUPerfStudio's hardcoding of
* performance counter group IDs.
*/
static const char * const si_pc_shader_type_suffixes[] = {
"", "_ES", "_GS", "_VS", "_PS", "_LS", "_HS", "_CS"
};
static const unsigned si_pc_shader_type_bits[] = {
0x7f,
S_036780_ES_EN(1),
S_036780_GS_EN(1),
S_036780_VS_EN(1),
S_036780_PS_EN(1),
S_036780_LS_EN(1),
S_036780_HS_EN(1),
S_036780_CS_EN(1),
};
+/* Max counters per HW block */
+#define SI_QUERY_MAX_COUNTERS 16
+
+#define SI_PC_SHADERS_WINDOWING (1 << 31)
+
+struct si_query_group {
+ struct si_query_group *next;
+ struct si_pc_block *block;
+ unsigned sub_gid; /* only used during init */
+ unsigned result_base; /* only used during init */
+ int se;
+ int instance;
+ unsigned num_counters;
+ unsigned selectors[SI_QUERY_MAX_COUNTERS];
+};
+
+struct si_query_counter {
+ unsigned base;
+ unsigned qwords;
+ unsigned stride; /* in uint64s */
+};
+
+struct si_query_pc {
+ struct si_query_hw b;
+
+ unsigned shaders;
+ unsigned num_counters;
+ struct si_query_counter *counters;
+ struct si_query_group *groups;
+};
+
+
static struct si_pc_block_base cik_CB = {
.name = "CB",
.num_counters = 4,
.flags = SI_PC_BLOCK_SE | SI_PC_BLOCK_INSTANCE_GROUPS,
.select0 = R_037000_CB_PERFCOUNTER_FILTER,
.counter0_lo = R_035018_CB_PERFCOUNTER0_LO,
.num_multi = 1,
.num_prelude = 1,
.layout = SI_PC_MULTI_ALTERNATE,
@@ -337,21 +399,21 @@ static struct si_pc_block_base cik_SRBM = {
};
/* Both the number of instances and selectors varies between chips of the same
* class. We only differentiate by class here and simply expose the maximum
* number over all chips in a class.
*
* Unfortunately, GPUPerfStudio uses the order of performance counter groups
* blindly once it believes it has identified the hardware, so the order of
* blocks here matters.
*/
-static struct si_pc_block groups_CIK[] = {
+static struct si_pc_block_gfxdescr groups_CIK[] = {
{ &cik_CB, 226},
{ &cik_CPF, 17 },
{ &cik_DB, 257},
{ &cik_GRBM, 34 },
{ &cik_GRBMSE, 15 },
{ &cik_PA_SU, 153 },
{ &cik_PA_SC, 395 },
{ &cik_SPI, 186 },
{ &cik_SQ, 252 },
{ &cik_SX, 32 },
@@ -364,21 +426,21 @@ static struct si_pc_block groups_CIK[] = {
{ &cik_VGT, 140 },
{ &cik_IA, 22 },
{ &cik_MC, 22 },
{ &cik_SRBM, 19 },
{ &cik_WD, 22 },
{ &cik_CPG, 46 },
{ &cik_CPC, 22 },
};
-static struct si_pc_block groups_VI[] = {
+static struct si_pc_block_gfxdescr groups_VI[] = {
{ &cik_CB, 405},
{ &cik_CPF, 19 },
{ &cik_DB, 257},
{ &cik_GRBM, 34 },
{ &cik_GRBMSE, 15 },
{ &cik_PA_SU, 154 },
{ &cik_PA_SC, 397 },
{ &cik_SPI, 197 },
{ &cik_SQ, 273 },
{ &cik_SX, 34 },
@@ -391,21 +453,21 @@ static struct si_pc_block groups_VI[] = {
{ &cik_VGT, 147 },
{ &cik_IA, 24 },
{ &cik_MC, 22 },
{ &cik_SRBM, 27 },
{ &cik_WD, 37 },
{ &cik_CPG, 48 },
{ &cik_CPC, 24 },
};
-static struct si_pc_block groups_gfx9[] = {
+static struct si_pc_block_gfxdescr groups_gfx9[] = {
{ &cik_CB, 438},
{ &cik_CPF, 32 },
{ &cik_DB, 328},
{ &cik_GRBM, 38 },
{ &cik_GRBMSE, 16 },
{ &cik_PA_SU, 292 },
{ &cik_PA_SC, 491 },
{ &cik_SPI, 196 },
{ &cik_SQ, 374 },
{ &cik_SX, 208 },
@@ -415,20 +477,72 @@ static struct si_pc_block groups_gfx9[] = {
{ &cik_TD, 57, 16 },
{ &cik_TCP, 85, 16 },
{ &cik_GDS, 121 },
{ &cik_VGT, 148 },
{ &cik_IA, 32 },
{ &cik_WD, 58 },
{ &cik_CPG, 59 },
{ &cik_CPC, 35 },
};
+static bool si_pc_block_has_per_se_groups(const struct si_perfcounters *pc,
+ const struct si_pc_block *block)
+{
+ return block->b->b->flags & SI_PC_BLOCK_SE_GROUPS ||
+ (block->b->b->flags & SI_PC_BLOCK_SE && pc->separate_se);
+}
+
+static bool si_pc_block_has_per_instance_groups(const struct si_perfcounters *pc,
+ const struct si_pc_block *block)
+{
+ return block->b->b->flags & SI_PC_BLOCK_INSTANCE_GROUPS ||
+ (block->num_instances > 1 && pc->separate_instance);
+}
+
+static struct si_pc_block *
+lookup_counter(struct si_perfcounters *pc, unsigned index,
+ unsigned *base_gid, unsigned *sub_index)
+{
+ struct si_pc_block *block = pc->blocks;
+ unsigned bid;
+
+ *base_gid = 0;
+ for (bid = 0; bid < pc->num_blocks; ++bid, ++block) {
+ unsigned total = block->num_groups * block->b->selectors;
+
+ if (index < total) {
+ *sub_index = index;
+ return block;
+ }
+
+ index -= total;
+ *base_gid += block->num_groups;
+ }
+
+ return NULL;
+}
+
+static struct si_pc_block *
+lookup_group(struct si_perfcounters *pc, unsigned *index)
+{
+ unsigned bid;
+ struct si_pc_block *block = pc->blocks;
+
+ for (bid = 0; bid < pc->num_blocks; ++bid, ++block) {
+ if (*index < block->num_groups)
+ return block;
+ *index -= block->num_groups;
+ }
+
+ return NULL;
+}
+
static void si_pc_emit_instance(struct si_context *sctx,
int se, int instance)
{
struct radeon_cmdbuf *cs = sctx->gfx_cs;
unsigned value = S_030800_SH_BROADCAST_WRITES(1);
if (se >= 0) {
value |= S_030800_SE_INDEX(se);
} else {
value |= S_030800_SE_BROADCAST_WRITES(1);
@@ -447,25 +561,24 @@ static void si_pc_emit_shaders(struct si_context *sctx,
unsigned shaders)
{
struct radeon_cmdbuf *cs = sctx->gfx_cs;
radeon_set_uconfig_reg_seq(cs, R_036780_SQ_PERFCOUNTER_CTRL, 2);
radeon_emit(cs, shaders & 0x7f);
radeon_emit(cs, 0xffffffff);
}
static void si_pc_emit_select(struct si_context *sctx,
- struct si_perfcounter_block *group,
+ struct si_pc_block *block,
unsigned count, unsigned *selectors)
{
- struct si_pc_block *sigroup = (struct si_pc_block *)group->data;
- struct si_pc_block_base *regs = sigroup->b;
+ struct si_pc_block_base *regs = block->b->b;
struct radeon_cmdbuf *cs = sctx->gfx_cs;
unsigned idx;
unsigned layout_multi = regs->layout & SI_PC_MULTI_MASK;
unsigned dw;
assert(count <= regs->num_counters);
if (regs->layout & SI_PC_FAKE)
return;
@@ -589,26 +702,24 @@ static void si_pc_emit_stop(struct si_context *sctx,
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_SAMPLE) | EVENT_INDEX(0));
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_STOP) | EVENT_INDEX(0));
radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
S_036020_PERFMON_STATE(V_036020_STOP_COUNTING) |
S_036020_PERFMON_SAMPLE_ENABLE(1));
}
static void si_pc_emit_read(struct si_context *sctx,
- struct si_perfcounter_block *group,
- unsigned count, unsigned *selectors,
- struct r600_resource *buffer, uint64_t va)
+ struct si_pc_block *block,
+ unsigned count, uint64_t va)
{
- struct si_pc_block *sigroup = (struct si_pc_block *)group->data;
- struct si_pc_block_base *regs = sigroup->b;
+ struct si_pc_block_base *regs = block->b->b;
struct radeon_cmdbuf *cs = sctx->gfx_cs;
unsigned idx;
unsigned reg = regs->counter0_lo;
unsigned reg_delta = 8;
if (!(regs->layout & SI_PC_FAKE)) {
if (regs->layout & SI_PC_REG_REVERSE)
reg_delta = -reg_delta;
for (idx = 0; idx < count; ++idx) {
@@ -634,30 +745,513 @@ static void si_pc_emit_read(struct si_context *sctx,
COPY_DATA_COUNT_SEL);
radeon_emit(cs, 0); /* immediate */
radeon_emit(cs, 0);
radeon_emit(cs, va);
radeon_emit(cs, va >> 32);
va += sizeof(uint64_t);
}
}
}
-static void si_pc_cleanup(struct si_screen *sscreen)
+static void si_pc_query_destroy(struct si_screen *sscreen,
+ struct si_query *rquery)
+{
+ struct si_query_pc *query = (struct si_query_pc *)rquery;
+
+ while (query->groups) {
+ struct si_query_group *group = query->groups;
+ query->groups = group->next;
+ FREE(group);
+ }
+
+ FREE(query->counters);
+
+ si_query_hw_destroy(sscreen, rquery);
+}
+
+static bool si_pc_query_prepare_buffer(struct si_screen *screen,
+ struct si_query_hw *hwquery,
+ struct r600_resource *buffer)
+{
+ /* no-op */
+ return true;
+}
+
+static void si_pc_query_emit_start(struct si_context *sctx,
+ struct si_query_hw *hwquery,
+ struct r600_resource *buffer, uint64_t va)
+{
+ struct si_query_pc *query = (struct si_query_pc *)hwquery;
+ struct si_query_group *group;
+ int current_se = -1;
+ int current_instance = -1;
+
+ if (query->shaders)
+ si_pc_emit_shaders(sctx, query->shaders);
+
+ for (group = query->groups; group; group = group->next) {
+ struct si_pc_block *block = group->block;
+
+ if (group->se != current_se || group->instance != current_instance) {
+ current_se = group->se;
+ current_instance = group->instance;
+ si_pc_emit_instance(sctx, group->se, group->instance);
+ }
+
+ si_pc_emit_select(sctx, block, group->num_counters, group->selectors);
+ }
+
+ if (current_se != -1 || current_instance != -1)
+ si_pc_emit_instance(sctx, -1, -1);
+
+ si_pc_emit_start(sctx, buffer, va);
+}
+
+static void si_pc_query_emit_stop(struct si_context *sctx,
+ struct si_query_hw *hwquery,
+ struct r600_resource *buffer, uint64_t va)
+{
+ struct si_query_pc *query = (struct si_query_pc *)hwquery;
+ struct si_query_group *group;
+
+ si_pc_emit_stop(sctx, buffer, va);
+
+ for (group = query->groups; group; group = group->next) {
+ struct si_pc_block *block = group->block;
+ unsigned se = group->se >= 0 ? group->se : 0;
+ unsigned se_end = se + 1;
+
+ if ((block->b->b->flags & SI_PC_BLOCK_SE) && (group->se < 0))
+ se_end = sctx->screen->info.max_se;
+
+ do {
+ unsigned instance = group->instance >= 0 ? group->instance : 0;
+
+ do {
+ si_pc_emit_instance(sctx, se, instance);
+ si_pc_emit_read(sctx, block, group->num_counters, va);
+ va += sizeof(uint64_t) * group->num_counters;
+ } while (group->instance < 0 && ++instance < block->num_instances);
+ } while (++se < se_end);
+ }
+
+ si_pc_emit_instance(sctx, -1, -1);
+}
+
+static void si_pc_query_clear_result(struct si_query_hw *hwquery,
+ union pipe_query_result *result)
+{
+ struct si_query_pc *query = (struct si_query_pc *)hwquery;
+
+ memset(result, 0, sizeof(result->batch[0]) * query->num_counters);
+}
+
+static void si_pc_query_add_result(struct si_screen *screen,
+ struct si_query_hw *hwquery,
+ void *buffer,
+ union pipe_query_result *result)
+{
+ struct si_query_pc *query = (struct si_query_pc *)hwquery;
+ uint64_t *results = buffer;
+ unsigned i, j;
+
+ for (i = 0; i < query->num_counters; ++i) {
+ struct si_query_counter *counter = &query->counters[i];
+
+ for (j = 0; j < counter->qwords; ++j) {
+ uint32_t value = results[counter->base + j * counter->stride];
+ result->batch[i].u64 += value;
+ }
+ }
+}
+
+static struct si_query_ops batch_query_ops = {
+ .destroy = si_pc_query_destroy,
+ .begin = si_query_hw_begin,
+ .end = si_query_hw_end,
+ .get_result = si_query_hw_get_result
+};
+
+static struct si_query_hw_ops batch_query_hw_ops = {
+ .prepare_buffer = si_pc_query_prepare_buffer,
+ .emit_start = si_pc_query_emit_start,
+ .emit_stop = si_pc_query_emit_stop,
+ .clear_result = si_pc_query_clear_result,
+ .add_result = si_pc_query_add_result,
+};
+
+static struct si_query_group *get_group_state(struct si_screen *screen,
+ struct si_query_pc *query,
+ struct si_pc_block *block,
+ unsigned sub_gid)
+{
+ struct si_query_group *group = query->groups;
+
+ while (group) {
+ if (group->block == block && group->sub_gid == sub_gid)
+ return group;
+ group = group->next;
+ }
+
+ group = CALLOC_STRUCT(si_query_group);
+ if (!group)
+ return NULL;
+
+ group->block = block;
+ group->sub_gid = sub_gid;
+
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER) {
+ unsigned sub_gids = block->num_instances;
+ unsigned shader_id;
+ unsigned shaders;
+ unsigned query_shaders;
+
+ if (si_pc_block_has_per_se_groups(screen->perfcounters, block))
+ sub_gids = sub_gids * screen->info.max_se;
+ shader_id = sub_gid / sub_gids;
+ sub_gid = sub_gid % sub_gids;
+
+ shaders = si_pc_shader_type_bits[shader_id];
+
+ query_shaders = query->shaders & ~SI_PC_SHADERS_WINDOWING;
+ if (query_shaders && query_shaders != shaders) {
+ fprintf(stderr, "si_perfcounter: incompatible shader groups\n");
+ FREE(group);
+ return NULL;
+ }
+ query->shaders = shaders;
+ }
+
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER_WINDOWED && !query->shaders) {
+ // A non-zero value in query->shaders ensures that the shader
+ // masking is reset unless the user explicitly requests one.
+ query->shaders = SI_PC_SHADERS_WINDOWING;
+ }
+
+ if (si_pc_block_has_per_se_groups(screen->perfcounters, block)) {
+ group->se = sub_gid / block->num_instances;
+ sub_gid = sub_gid % block->num_instances;
+ } else {
+ group->se = -1;
+ }
+
+ if (si_pc_block_has_per_instance_groups(screen->perfcounters, block)) {
+ group->instance = sub_gid;
+ } else {
+ group->instance = -1;
+ }
+
+ group->next = query->groups;
+ query->groups = group;
+
+ return group;
+}
+
+struct pipe_query *si_create_batch_query(struct pipe_context *ctx,
+ unsigned num_queries,
+ unsigned *query_types)
+{
+ struct si_screen *screen =
+ (struct si_screen *)ctx->screen;
+ struct si_perfcounters *pc = screen->perfcounters;
+ struct si_pc_block *block;
+ struct si_query_group *group;
+ struct si_query_pc *query;
+ unsigned base_gid, sub_gid, sub_index;
+ unsigned i, j;
+
+ if (!pc)
+ return NULL;
+
+ query = CALLOC_STRUCT(si_query_pc);
+ if (!query)
+ return NULL;
+
+ query->b.b.ops = &batch_query_ops;
+ query->b.ops = &batch_query_hw_ops;
+
+ query->num_counters = num_queries;
+
+ /* Collect selectors per group */
+ for (i = 0; i < num_queries; ++i) {
+ unsigned sub_gid;
+
+ if (query_types[i] < SI_QUERY_FIRST_PERFCOUNTER)
+ goto error;
+
+ block = lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER,
+ &base_gid, &sub_index);
+ if (!block)
+ goto error;
+
+ sub_gid = sub_index / block->b->selectors;
+ sub_index = sub_index % block->b->selectors;
+
+ group = get_group_state(screen, query, block, sub_gid);
+ if (!group)
+ goto error;
+
+ if (group->num_counters >= block->b->b->num_counters) {
+ fprintf(stderr,
+ "perfcounter group %s: too many selected\n",
+ block->b->b->name);
+ goto error;
+ }
+ group->selectors[group->num_counters] = sub_index;
+ ++group->num_counters;
+ }
+
+ /* Compute result bases and CS size per group */
+ query->b.num_cs_dw_end = pc->num_stop_cs_dwords;
+ query->b.num_cs_dw_end += pc->num_instance_cs_dwords;
+
+ i = 0;
+ for (group = query->groups; group; group = group->next) {
+ struct si_pc_block *block = group->block;
+ unsigned read_dw;
+ unsigned instances = 1;
+
+ if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0)
+ instances = screen->info.max_se;
+ if (group->instance < 0)
+ instances *= block->num_instances;
+
+ group->result_base = i;
+ query->b.result_size += sizeof(uint64_t) * instances * group->num_counters;
+ i += instances * group->num_counters;
+
+ read_dw = 6 * group->num_counters;
+ query->b.num_cs_dw_end += instances * read_dw;
+ query->b.num_cs_dw_end += instances * pc->num_instance_cs_dwords;
+ }
+
+ if (query->shaders) {
+ if (query->shaders == SI_PC_SHADERS_WINDOWING)
+ query->shaders = 0xffffffff;
+ }
+
+ /* Map user-supplied query array to result indices */
+ query->counters = CALLOC(num_queries, sizeof(*query->counters));
+ for (i = 0; i < num_queries; ++i) {
+ struct si_query_counter *counter = &query->counters[i];
+ struct si_pc_block *block;
+
+ block = lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER,
+ &base_gid, &sub_index);
+
+ sub_gid = sub_index / block->b->selectors;
+ sub_index = sub_index % block->b->selectors;
+
+ group = get_group_state(screen, query, block, sub_gid);
+ assert(group != NULL);
+
+ for (j = 0; j < group->num_counters; ++j) {
+ if (group->selectors[j] == sub_index)
+ break;
+ }
+
+ counter->base = group->result_base + j;
+ counter->stride = group->num_counters;
+
+ counter->qwords = 1;
+ if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0)
+ counter->qwords = screen->info.max_se;
+ if (group->instance < 0)
+ counter->qwords *= block->num_instances;
+ }
+
+ if (!si_query_hw_init(screen, &query->b))
+ goto error;
+
+ return (struct pipe_query *)query;
+
+error:
+ si_pc_query_destroy(screen, &query->b.b);
+ return NULL;
+}
+
+static bool si_init_block_names(struct si_screen *screen,
+ struct si_pc_block *block)
+{
+ bool per_instance_groups = si_pc_block_has_per_instance_groups(screen->perfcounters, block);
+ bool per_se_groups = si_pc_block_has_per_se_groups(screen->perfcounters, block);
+ unsigned i, j, k;
+ unsigned groups_shader = 1, groups_se = 1, groups_instance = 1;
+ unsigned namelen;
+ char *groupname;
+ char *p;
+
+ if (per_instance_groups)
+ groups_instance = block->num_instances;
+ if (per_se_groups)
+ groups_se = screen->info.max_se;
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER)
+ groups_shader = ARRAY_SIZE(si_pc_shader_type_bits);
+
+ namelen = strlen(block->b->b->name);
+ block->group_name_stride = namelen + 1;
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER)
+ block->group_name_stride += 3;
+ if (per_se_groups) {
+ assert(groups_se <= 10);
+ block->group_name_stride += 1;
+
+ if (per_instance_groups)
+ block->group_name_stride += 1;
+ }
+ if (per_instance_groups) {
+ assert(groups_instance <= 100);
+ block->group_name_stride += 2;
+ }
+
+ block->group_names = MALLOC(block->num_groups * block->group_name_stride);
+ if (!block->group_names)
+ return false;
+
+ groupname = block->group_names;
+ for (i = 0; i < groups_shader; ++i) {
+ const char *shader_suffix = si_pc_shader_type_suffixes[i];
+ unsigned shaderlen = strlen(shader_suffix);
+ for (j = 0; j < groups_se; ++j) {
+ for (k = 0; k < groups_instance; ++k) {
+ strcpy(groupname, block->b->b->name);
+ p = groupname + namelen;
+
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER) {
+ strcpy(p, shader_suffix);
+ p += shaderlen;
+ }
+
+ if (per_se_groups) {
+ p += sprintf(p, "%d", j);
+ if (per_instance_groups)
+ *p++ = '_';
+ }
+
+ if (per_instance_groups)
+ p += sprintf(p, "%d", k);
+
+ groupname += block->group_name_stride;
+ }
+ }
+ }
+
+ assert(block->b->selectors <= 1000);
+ block->selector_name_stride = block->group_name_stride + 4;
+ block->selector_names = MALLOC(block->num_groups * block->b->selectors *
+ block->selector_name_stride);
+ if (!block->selector_names)
+ return false;
+
+ groupname = block->group_names;
+ p = block->selector_names;
+ for (i = 0; i < block->num_groups; ++i) {
+ for (j = 0; j < block->b->selectors; ++j) {
+ sprintf(p, "%s_%03d", groupname, j);
+ p += block->selector_name_stride;
+ }
+ groupname += block->group_name_stride;
+ }
+
+ return true;
+}
+
+int si_get_perfcounter_info(struct si_screen *screen,
+ unsigned index,
+ struct pipe_driver_query_info *info)
+{
+ struct si_perfcounters *pc = screen->perfcounters;
+ struct si_pc_block *block;
+ unsigned base_gid, sub;
+
+ if (!pc)
+ return 0;
+
+ if (!info) {
+ unsigned bid, num_queries = 0;
+
+ for (bid = 0; bid < pc->num_blocks; ++bid) {
+ num_queries += pc->blocks[bid].b->selectors *
+ pc->blocks[bid].num_groups;
+ }
+
+ return num_queries;
+ }
+
+ block = lookup_counter(pc, index, &base_gid, &sub);
+ if (!block)
+ return 0;
+
+ if (!block->selector_names) {
+ if (!si_init_block_names(screen, block))
+ return 0;
+ }
+ info->name = block->selector_names + sub * block->selector_name_stride;
+ info->query_type = SI_QUERY_FIRST_PERFCOUNTER + index;
+ info->max_value.u64 = 0;
+ info->type = PIPE_DRIVER_QUERY_TYPE_UINT64;
+ info->result_type = PIPE_DRIVER_QUERY_RESULT_TYPE_AVERAGE;
+ info->group_id = base_gid + sub / block->b->selectors;
+ info->flags = PIPE_DRIVER_QUERY_FLAG_BATCH;
+ if (sub > 0 && sub + 1 < block->b->selectors * block->num_groups)
+ info->flags |= PIPE_DRIVER_QUERY_FLAG_DONT_LIST;
+ return 1;
+}
+
+int si_get_perfcounter_group_info(struct si_screen *screen,
+ unsigned index,
+ struct pipe_driver_query_group_info *info)
{
- si_perfcounters_do_destroy(sscreen->perfcounters);
- sscreen->perfcounters = NULL;
+ struct si_perfcounters *pc = screen->perfcounters;
+ struct si_pc_block *block;
+
+ if (!pc)
+ return 0;
+
+ if (!info)
+ return pc->num_groups;
+
+ block = lookup_group(pc, &index);
+ if (!block)
+ return 0;
+
+ if (!block->group_names) {
+ if (!si_init_block_names(screen, block))
+ return 0;
+ }
+ info->name = block->group_names + index * block->group_name_stride;
+ info->num_queries = block->b->selectors;
+ info->max_active_queries = block->b->b->num_counters;
+ return 1;
+}
+
+void si_destroy_perfcounters(struct si_screen *screen)
+{
+ struct si_perfcounters *pc = screen->perfcounters;
+ unsigned i;
+
+ if (!pc)
+ return;
+
+ for (i = 0; i < pc->num_blocks; ++i) {
+ FREE(pc->blocks[i].group_names);
+ FREE(pc->blocks[i].selector_names);
+ }
+ FREE(pc->blocks);
+ FREE(pc);
+ screen->perfcounters = NULL;
}
void si_init_perfcounters(struct si_screen *screen)
{
struct si_perfcounters *pc;
- struct si_pc_block *blocks;
+ const struct si_pc_block_gfxdescr *blocks;
unsigned num_blocks;
unsigned i;
switch (screen->info.chip_class) {
case CIK:
blocks = groups_CIK;
num_blocks = ARRAY_SIZE(groups_CIK);
break;
case VI:
blocks = groups_VI;
@@ -672,59 +1266,57 @@ void si_init_perfcounters(struct si_screen *screen)
return; /* not implemented */
}
if (screen->info.max_sh_per_se != 1) {
/* This should not happen on non-SI chips. */
fprintf(stderr, "si_init_perfcounters: max_sh_per_se = %d not "
"supported (inaccurate performance counters)\n",
screen->info.max_sh_per_se);
}
- pc = CALLOC_STRUCT(si_perfcounters);
+ screen->perfcounters = pc = CALLOC_STRUCT(si_perfcounters);
if (!pc)
return;
pc->num_stop_cs_dwords = 14 + si_cp_write_fence_dwords(screen);
pc->num_instance_cs_dwords = 3;
- pc->num_shader_types = ARRAY_SIZE(si_pc_shader_type_bits);
- pc->shader_type_suffixes = si_pc_shader_type_suffixes;
- pc->shader_type_bits = si_pc_shader_type_bits;
-
- pc->emit_instance = si_pc_emit_instance;
- pc->emit_shaders = si_pc_emit_shaders;
- pc->emit_select = si_pc_emit_select;
- pc->emit_start = si_pc_emit_start;
- pc->emit_stop = si_pc_emit_stop;
- pc->emit_read = si_pc_emit_read;
- pc->cleanup = si_pc_cleanup;
+ pc->separate_se = debug_get_bool_option("RADEON_PC_SEPARATE_SE", false);
+ pc->separate_instance = debug_get_bool_option("RADEON_PC_SEPARATE_INSTANCE", false);
- if (!si_perfcounters_init(pc, num_blocks))
+ pc->blocks = CALLOC(num_blocks, sizeof(struct si_pc_block));
+ if (!pc->blocks)
goto error;
+ pc->num_blocks = num_blocks;
for (i = 0; i < num_blocks; ++i) {
- struct si_pc_block *block = &blocks[i];
- unsigned instances = block->instances;
+ struct si_pc_block *block = &pc->blocks[i];
+ block->b = &blocks[i];
+ block->num_instances = block->b->instances;
+
+ if (!strcmp(block->b->b->name, "CB") ||
+ !strcmp(block->b->b->name, "DB"))
+ block->num_instances = screen->info.max_se;
+ else if (!strcmp(block->b->b->name, "TCC"))
+ block->num_instances = screen->info.num_tcc_blocks;
+ else if (!strcmp(block->b->b->name, "IA"))
+ block->num_instances = MAX2(1, screen->info.max_se / 2);
+
+ if (si_pc_block_has_per_instance_groups(pc, block)) {
+ block->num_groups = block->num_instances;
+ } else {
+ block->num_groups = 1;
+ }
- if (!strcmp(block->b->name, "CB") ||
- !strcmp(block->b->name, "DB"))
- instances = screen->info.max_se;
- else if (!strcmp(block->b->name, "TCC"))
- instances = screen->info.num_tcc_blocks;
- else if (!strcmp(block->b->name, "IA"))
- instances = MAX2(1, screen->info.max_se / 2);
-
- si_perfcounters_add_block(screen, pc,
- block->b->name,
- block->b->flags,
- block->b->num_counters,
- block->selectors,
- instances,
- block);
+ if (si_pc_block_has_per_se_groups(pc, block))
+ block->num_groups *= screen->info.max_se;
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER)
+ block->num_groups *= ARRAY_SIZE(si_pc_shader_type_bits);
+
+ pc->num_groups += block->num_groups;
}
- screen->perfcounters = pc;
return;
error:
- si_perfcounters_do_destroy(pc);
+ si_destroy_perfcounters(screen);
}
diff --git a/src/gallium/drivers/radeonsi/si_pipe.c b/src/gallium/drivers/radeonsi/si_pipe.c
index fd8ff5fa202..f7bab2c59f5 100644
--- a/src/gallium/drivers/radeonsi/si_pipe.c
+++ b/src/gallium/drivers/radeonsi/si_pipe.c
@@ -698,21 +698,21 @@ static void si_destroy_screen(struct pipe_screen* pscreen)
struct si_shader_part *part = parts[i];
parts[i] = part->next;
ac_shader_binary_clean(&part->binary);
FREE(part);
}
}
mtx_destroy(&sscreen->shader_parts_mutex);
si_destroy_shader_cache(sscreen);
- si_perfcounters_destroy(sscreen);
+ si_destroy_perfcounters(sscreen);
si_gpu_load_kill_thread(sscreen);
mtx_destroy(&sscreen->gpu_load_mutex);
mtx_destroy(&sscreen->aux_context_lock);
sscreen->aux_context->destroy(sscreen->aux_context);
slab_destroy_parent(&sscreen->pool_transfers);
disk_cache_destroy(sscreen->disk_shader_cache);
sscreen->ws->destroy(sscreen->ws);
diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h
index 179671e8871..bfcfc915124 100644
--- a/src/gallium/drivers/radeonsi/si_pipe.h
+++ b/src/gallium/drivers/radeonsi/si_pipe.h
@@ -1229,25 +1229,23 @@ void si_need_gfx_cs_space(struct si_context *ctx);
/* r600_gpu_load.c */
void si_gpu_load_kill_thread(struct si_screen *sscreen);
uint64_t si_begin_counter(struct si_screen *sscreen, unsigned type);
unsigned si_end_counter(struct si_screen *sscreen, unsigned type,
uint64_t begin);
/* si_compute.c */
void si_init_compute_functions(struct si_context *sctx);
-/* r600_perfcounters.c */
-void si_perfcounters_destroy(struct si_screen *sscreen);
-
/* si_perfcounters.c */
void si_init_perfcounters(struct si_screen *screen);
+void si_destroy_perfcounters(struct si_screen *screen);
/* si_pipe.c */
bool si_check_device_reset(struct si_context *sctx);
/* si_query.c */
void si_init_screen_query_functions(struct si_screen *sscreen);
void si_init_query_functions(struct si_context *sctx);
void si_suspend_queries(struct si_context *sctx);
void si_resume_queries(struct si_context *sctx);
diff --git a/src/gallium/drivers/radeonsi/si_query.h b/src/gallium/drivers/radeonsi/si_query.h
index cf2eccd862b..032946edf4d 100644
--- a/src/gallium/drivers/radeonsi/si_query.h
+++ b/src/gallium/drivers/radeonsi/si_query.h
@@ -206,115 +206,43 @@ void si_query_hw_destroy(struct si_screen *sscreen,
bool si_query_hw_begin(struct si_context *sctx,
struct si_query *rquery);
bool si_query_hw_end(struct si_context *sctx,
struct si_query *rquery);
bool si_query_hw_get_result(struct si_context *sctx,
struct si_query *rquery,
bool wait,
union pipe_query_result *result);
/* Performance counters */
-enum {
- /* This block is part of the shader engine */
- SI_PC_BLOCK_SE = (1 << 0),
-
- /* Expose per-instance groups instead of summing all instances (within
- * an SE). */
- SI_PC_BLOCK_INSTANCE_GROUPS = (1 << 1),
-
- /* Expose per-SE groups instead of summing instances across SEs. */
- SI_PC_BLOCK_SE_GROUPS = (1 << 2),
-
- /* Shader block */
- SI_PC_BLOCK_SHADER = (1 << 3),
-
- /* Non-shader block with perfcounters windowed by shaders. */
- SI_PC_BLOCK_SHADER_WINDOWED = (1 << 4),
-};
-
-/* Describes a hardware block with performance counters. Multiple instances of
- * each block, possibly per-SE, may exist on the chip. Depending on the block
- * and on the user's configuration, we either
- * (a) expose every instance as a performance counter group,
- * (b) expose a single performance counter group that reports the sum over all
- * instances, or
- * (c) expose one performance counter group per instance, but summed over all
- * shader engines.
- */
-struct si_perfcounter_block {
- const char *basename;
- unsigned flags;
- unsigned num_counters;
- unsigned num_selectors;
- unsigned num_instances;
-
- unsigned num_groups;
- char *group_names;
- unsigned group_name_stride;
-
- char *selector_names;
- unsigned selector_name_stride;
-
- void *data;
-};
-
struct si_perfcounters {
unsigned num_groups;
unsigned num_blocks;
- struct si_perfcounter_block *blocks;
+ struct si_pc_block *blocks;
unsigned num_stop_cs_dwords;
unsigned num_instance_cs_dwords;
- unsigned num_shader_types;
- const char * const *shader_type_suffixes;
- const unsigned *shader_type_bits;
-
- void (*emit_instance)(struct si_context *,
- int se, int instance);
- void (*emit_shaders)(struct si_context *, unsigned shaders);
- void (*emit_select)(struct si_context *,
- struct si_perfcounter_block *,
- unsigned count, unsigned *selectors);
- void (*emit_start)(struct si_context *,
- struct r600_resource *buffer, uint64_t va);
- void (*emit_stop)(struct si_context *,
- struct r600_resource *buffer, uint64_t va);
- void (*emit_read)(struct si_context *,
- struct si_perfcounter_block *,
- unsigned count, unsigned *selectors,
- struct r600_resource *buffer, uint64_t va);
-
- void (*cleanup)(struct si_screen *);
-
bool separate_se;
bool separate_instance;
};
struct pipe_query *si_create_batch_query(struct pipe_context *ctx,
unsigned num_queries,
unsigned *query_types);
int si_get_perfcounter_info(struct si_screen *,
unsigned index,
struct pipe_driver_query_info *info);
int si_get_perfcounter_group_info(struct si_screen *,
unsigned index,
struct pipe_driver_query_group_info *info);
-bool si_perfcounters_init(struct si_perfcounters *, unsigned num_blocks);
-void si_perfcounters_add_block(struct si_screen *,
- struct si_perfcounters *,
- const char *name, unsigned flags,
- unsigned counters, unsigned selectors,
- unsigned instances, void *data);
-void si_perfcounters_do_destroy(struct si_perfcounters *);
void si_query_hw_reset_buffers(struct si_context *sctx,
struct si_query_hw *query);
struct si_qbo_state {
void *saved_compute;
struct pipe_constant_buffer saved_const0;
struct pipe_shader_buffer saved_ssbo[3];
};
#endif /* SI_QUERY_H */
--
2.19.1