diff --git a/pkg/controller/scheduledscaling/scheduled_scaling.go b/pkg/controller/scheduledscaling/scheduled_scaling.go
index d1b38aa..34ceb51 100644
--- a/pkg/controller/scheduledscaling/scheduled_scaling.go
+++ b/pkg/controller/scheduledscaling/scheduled_scaling.go
@@ -320,6 +320,10 @@ func highestActiveSchedule(hpa *autoscalingv2.HorizontalPodAutoscaler, activeSch
 
 		scheduleName := metric.Object.DescribedObject.Name
 
+		if metric.Object.Target.AverageValue == nil {
+			continue
+		}
+
 		target := int64(metric.Object.Target.AverageValue.MilliValue() / 1000)
 		if target == 0 {
 			continue
diff --git a/pkg/controller/scheduledscaling/scheduled_scaling_test.go b/pkg/controller/scheduledscaling/scheduled_scaling_test.go
index b379f9c..59c86dc 100644
--- a/pkg/controller/scheduledscaling/scheduled_scaling_test.go
+++ b/pkg/controller/scheduledscaling/scheduled_scaling_test.go
@@ -350,6 +350,12 @@ func TestAdjustScaling(t *testing.T) {
 			desiredReplicas: 90,
 			targetValue:     10, // 1000/10 = 100
 		},
+		{
+			msg:             "invalid HPA should not do any adjustment",
+			currentReplicas: 95,
+			desiredReplicas: 95,
+			targetValue:     0, // this is treated as invalid in the test, thus the HPA is ingored and no adjustment happens.
+		},
 	} {
 		t.Run(tc.msg, func(t *testing.T) {
 			kubeClient := fake.NewSimpleClientset()
@@ -419,8 +425,7 @@ func TestAdjustScaling(t *testing.T) {
 									Name:       "schedule-1",
 								},
 								Target: v2.MetricTarget{
-									Type:         v2.AverageValueMetricType,
-									AverageValue: resource.NewQuantity(tc.targetValue, resource.DecimalSI),
+									Type: v2.AverageValueMetricType,
 								},
 							},
 						},
@@ -428,6 +433,10 @@ func TestAdjustScaling(t *testing.T) {
 				},
 			}
 
+			if tc.targetValue != 0 {
+				hpa.Spec.Metrics[0].Object.Target.AverageValue = resource.NewQuantity(tc.targetValue, resource.DecimalSI)
+			}
+
 			hpa, err = kubeClient.AutoscalingV2().HorizontalPodAutoscalers("default").Create(context.Background(), hpa, metav1.CreateOptions{})
 			require.NoError(t, err)