Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 108 additions & 1 deletion fault-remediation/pkg/annotation/annotation.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,11 @@ func (m *NodeAnnotationManager) GetRemediationState(

// UpdateRemediationState updates the node annotation with new remediation state
func (m *NodeAnnotationManager) UpdateRemediationState(ctx context.Context, nodeName string,
group string, crName string, actionName string) error {
group string, crName string, actionName string, resourceRef MaintenanceResourceReference) error {
if err := validateMaintenanceResourceReference(resourceRef); err != nil {
return fmt.Errorf("invalid maintenance resource reference for node %s group %s: %w", nodeName, group, err)
}

err := retry.RetryOnConflict(conflictBackoff, func() error {
// Get current state
state, node, err := m.GetRemediationState(ctx, nodeName)
Expand All @@ -109,6 +113,10 @@ func (m *NodeAnnotationManager) UpdateRemediationState(ctx context.Context, node
MaintenanceCR: crName,
CreatedAt: time.Now().UTC(),
ActionName: actionName,
Namespace: resourceRef.Namespace,
Version: resourceRef.Version,
APIGroup: resourceRef.APIGroup,
Kind: resourceRef.Kind,
}

// Marshal to JSON
Expand Down Expand Up @@ -142,6 +150,105 @@ func (m *NodeAnnotationManager) UpdateRemediationState(ctx context.Context, node
return nil
}

// EnsureRemediationStateGVK backfills concrete resource identity for legacy
// annotation entries using resourceRefsByAction, if and only if the GVK is not already set.
// The map must be keyed by ActionName.
func (m *NodeAnnotationManager) EnsureRemediationStateGVK(
ctx context.Context,
nodeName string,
resourceRefsByAction map[string]MaintenanceResourceReference,
) error {
err := retry.RetryOnConflict(conflictBackoff, func() error {
state, node, err := m.GetRemediationState(ctx, nodeName)
if err != nil {
slog.WarnContext(ctx, "Failed to get current remediation state", "node", nodeName, "error", err)
return err
}

changed := false

for group, groupState := range state.EquivalenceGroups {
resourceRef, exists := resourceRefsByAction[groupState.ActionName]
if !exists {
continue
}

updatedGroupState := groupState
if backfillResourceReference(&updatedGroupState, resourceRef) {
state.EquivalenceGroups[group] = updatedGroupState
changed = true
}
}

if !changed {
return nil
}

stateJSON, err := json.Marshal(state)
if err != nil {
return err
}

updatedNode := node.DeepCopy()
if updatedNode.Annotations == nil {
updatedNode.Annotations = map[string]string{}
}

updatedNode.Annotations[AnnotationKey] = string(stateJSON)

if err = m.client.Update(ctx, updatedNode); err != nil {
return err
}

slog.InfoContext(ctx, "Backfilled remediation state annotation GVK for node",
"node", nodeName)

return nil
})
if err != nil {
return fmt.Errorf("failed to ensure remediation state GVK for node %s: %w", nodeName, err)
}

return nil
}

func validateMaintenanceResourceReference(resourceRef MaintenanceResourceReference) error {
if resourceRef.APIGroup == "" || resourceRef.Version == "" || resourceRef.Kind == "" {
return fmt.Errorf("apiGroup, version, and kind must be non-empty")
}

return nil
}

func backfillResourceReference(
groupState *EquivalenceGroupState,
resourceRef MaintenanceResourceReference,
) bool {
changed := false

if groupState.APIGroup == "" && resourceRef.APIGroup != "" {
groupState.APIGroup = resourceRef.APIGroup
changed = true
}

if groupState.Version == "" && resourceRef.Version != "" {
groupState.Version = resourceRef.Version
changed = true
}

if groupState.Kind == "" && resourceRef.Kind != "" {
groupState.Kind = resourceRef.Kind
changed = true
}

if groupState.Namespace == "" && resourceRef.Namespace != "" {
groupState.Namespace = resourceRef.Namespace
changed = true
}

return changed
}

// ClearRemediationState removes the remediation state annotation from a node
func (m *NodeAnnotationManager) ClearRemediationState(ctx context.Context, nodeName string) error {
err := retry.RetryOnConflict(conflictBackoff, func() error {
Expand Down
35 changes: 32 additions & 3 deletions fault-remediation/pkg/annotation/annotation_interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,21 @@ const (
// NodeAnnotationManagerInterface defines the interface for managing node annotations
type NodeAnnotationManagerInterface interface {
GetRemediationState(ctx context.Context, nodeName string) (*RemediationStateAnnotation, *corev1.Node, error)
UpdateRemediationState(ctx context.Context, nodeName string, group string, crName string, actionName string) error
UpdateRemediationState(
ctx context.Context,
nodeName string,
group string,
crName string,
actionName string,
resourceRef MaintenanceResourceReference,
) error
// EnsureRemediationStateGVK backfills legacy annotation entries using
// resource references keyed by ActionName.
EnsureRemediationStateGVK(
ctx context.Context,
nodeName string,
resourceRefsByAction map[string]MaintenanceResourceReference,
) error
Comment thread
coderabbitai[bot] marked this conversation as resolved.
ClearRemediationState(ctx context.Context, nodeName string) error
RemoveGroupsFromState(ctx context.Context, nodeName string, groups []string) error
}
Expand All @@ -39,12 +53,27 @@ type RemediationStateAnnotation struct {
EquivalenceGroups map[string]EquivalenceGroupState `json:"equivalenceGroups"`
}

// MaintenanceResourceReference is the concrete object identity needed to find
// a created maintenance CR without re-deriving its GVK from current config.
type MaintenanceResourceReference struct {
Namespace string
Version string
APIGroup string
Kind string
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}

// EquivalenceGroupState represents the state of a single equivalence group
type EquivalenceGroupState struct {
MaintenanceCR string `json:"maintenanceCR"`
CreatedAt time.Time `json:"createdAt"`

// Action that created the CR (e.g., "RESTART_BM")
// Required to look up the corresponding MaintenanceResource from the TomlConfig
// Action that created the CR (e.g., "RESTART_BM"). Kept as provenance
// and to resolve legacy annotations that predate the concrete resource fields.
ActionName string `json:"actionName"`

// Concrete resource identity for the CR that was created.
Namespace string `json:"namespace,omitempty"`
Version string `json:"version,omitempty"`
APIGroup string `json:"apiGroup,omitempty"`
Kind string `json:"kind,omitempty"`
}
Loading